diff --git a/Dockerfile.prod.go b/Dockerfile.prod.go index 02727e00..14d8d174 100644 --- a/Dockerfile.prod.go +++ b/Dockerfile.prod.go @@ -9,11 +9,13 @@ RUN go install github.com/sqlc-dev/sqlc/cmd/sqlc@latest RUN sqlc generate RUN go mod tidy RUN CGO_ENABLED=0 GOOS=linux go build -a -installsuffix cgo -o server cmd/server/main.go +RUN CGO_ENABLED=0 GOOS=linux go build -a -installsuffix cgo -o vault cmd/vault/main.go FROM alpine:latest RUN apk --no-cache add ca-certificates tzdata WORKDIR /root/ COPY --from=builder /app/server . +COPY --from=builder /app/vault . COPY go-app/templates ./templates COPY go-app/static ./static COPY go-app/.env.example .env diff --git a/docker-compose.prod.yml b/docker-compose.prod.yml index 61047519..aab24865 100644 --- a/docker-compose.prod.yml +++ b/docker-compose.prod.yml @@ -81,6 +81,9 @@ services: - "8083:8082" volumes: - /home/cmc/files/pdf:/root/webroot/pdf:ro + - /home/cmc/files/emails:/var/www/emails + - /home/cmc/files/vault:/var/www/vault + - /home/cmc/files/vaultmsgs:/var/www/vaultmsgs networks: - cmc-prod-network restart: unless-stopped diff --git a/scripts/VAULT_DEPLOYMENT.md b/scripts/VAULT_DEPLOYMENT.md new file mode 100644 index 00000000..2bee4246 --- /dev/null +++ b/scripts/VAULT_DEPLOYMENT.md @@ -0,0 +1,231 @@ +# Go Vault Deployment Guide + +## Running the Go Vault in Docker Production + +### Prerequisites + +- The vault binary is built into the Docker image +- Vault directories are mounted from `/home/cmc/files/vaultmsgs/` +- Container has access to the database + +### Step 1: Rebuild the Docker Image + +The Dockerfile has been updated to include the vault binary. Rebuild the image: + +```bash +# On the production server +cd ~/src/cmc-sales + +# Rebuild the Go container +docker compose -f docker-compose.prod.yml build cmc-prod-go + +# Restart the container +docker compose -f docker-compose.prod.yml up -d cmc-prod-go +``` + +### Step 2: Verify Vault Binary Exists + +```bash +# Check if vault binary is in the container +docker exec -t cmc-prod-go ls -lh /root/vault + +# Test running vault (dry run) +docker exec -t cmc-prod-go ./vault --mode=local \ + --vaultdir=/var/www/vaultmsgs/new \ + --processeddir=/var/www/vaultmsgs/cur \ + --emaildir=/var/www/emails \ + --dbhost=cmc-prod-db \ + --dbuser=cmc \ + --dbpass="xVRQI&cA?7AU=hqJ!%au" \ + --dbname=cmc +``` + +### Step 3: Set Up Cron Job + +```bash +# Copy the cron script to the server +scp scripts/vault-cron-prod.sh cmc@sales.cmctechnologies.com.au:~/scripts/ + +# On the server, make it executable +chmod +x ~/scripts/vault-cron-prod.sh + +# Create log directory +mkdir -p ~/logs + +# Test the script manually +~/scripts/vault-cron-prod.sh + +# Check the log +tail -f ~/logs/vault-go.log +``` + +### Step 4: Add to Crontab + +```bash +# Edit crontab +crontab -e + +# Add this line to run every 5 minutes +*/5 * * * * /home/cmc/scripts/vault-cron-prod.sh +``` + +### Step 5: Monitor + +```bash +# Watch logs in real-time +tail -f ~/logs/vault-go.log + +# Check cron execution +grep vault /var/log/syslog + +# List cron jobs +crontab -l +``` + +## Directory Structure + +The following directories need to exist on the host: + +``` +/home/cmc/files/ +├── vaultmsgs/ +│ ├── new/ # Incoming emails to process +│ └── cur/ # Processed emails archive +├── emails/ # Email storage +└── vault/ # Additional vault data +``` + +## Troubleshooting + +### Vault binary not found + +```bash +# Rebuild the image +docker compose -f docker-compose.prod.yml build cmc-prod-go +docker compose -f docker-compose.prod.yml up -d cmc-prod-go +``` + +### Permission errors + +```bash +# Check directory permissions on host +ls -ld /home/cmc/files/vaultmsgs/new +ls -ld /home/cmc/files/vaultmsgs/cur + +# Should be readable/writable by the user running Docker +``` + +### Database connection errors + +```bash +# Verify database container is running +docker ps | grep cmc-prod-db + +# Test database connection from Go container +docker exec -it cmc-prod-go sh +nc -zv cmc-prod-db 3306 +``` + +### No emails being processed + +```bash +# Check if there are emails to process +ls -la /home/cmc/files/vaultmsgs/new/ + +# Check if getmail or other email fetcher is running +ps aux | grep getmail +``` + +## Switching from PHP to Go Vault + +### Current PHP Setup + +The PHP vault runs via: +```bash +/var/www/cmc-sales/cake/console/cake -app /var/www/cmc-sales/app vault +``` + +### Migration Steps + +1. **Run both in parallel** (testing phase): + ```bash + # Keep PHP vault running + */5 * * * * /path/to/old/vault_cron.sh + + # Add Go vault (different schedule) + */10 * * * * /home/cmc/scripts/vault-cron-prod.sh + ``` + +2. **Compare results**: + - Check database for duplicate emails + - Verify all business entities are linked correctly + - Compare attachment handling + +3. **Switch completely**: + ```bash + # Disable PHP vault + crontab -e # Comment out the PHP vault line + + # Enable Go vault only + */5 * * * * /home/cmc/scripts/vault-cron-prod.sh + ``` + +## Advanced: Gmail Mode + +For better performance, consider switching to Gmail API mode: + +### Setup Gmail OAuth (one-time) + +```bash +# Copy credentials to the server +scp credentials.json cmc@sales.cmctechnologies.com.au:~/ + +# Copy to container +docker cp ~/credentials.json cmc-prod-go:/root/credentials.json + +# Run initial authorization (interactive) +docker exec -it cmc-prod-go ./vault --mode=index \ + --credentials=credentials.json \ + --token=token.json \ + --gmail-query="is:unread newer_than:1d" \ + --dbhost=cmc-prod-db \ + --dbuser=cmc \ + --dbpass="xVRQI&cA?7AU=hqJ!%au" \ + --dbname=cmc + +# This will generate a URL - open in browser and authorize +# Token will be saved in container + +# Copy token back to host for backup +docker cp cmc-prod-go:/root/token.json ~/token.json +``` + +### Update Cron for Gmail Mode + +Edit [scripts/vault-cron-prod.sh](scripts/vault-cron-prod.sh) and change: + +```bash +docker exec -t "$CONTAINER_NAME" ./vault --mode=index \ + --credentials=credentials.json \ + --token=token.json \ + --gmail-query="is:unread newer_than:1d" \ + --dbhost=cmc-prod-db \ + --dbuser=cmc \ + --dbpass="xVRQI&cA?7AU=hqJ!%au" \ + --dbname=cmc \ + >> "$LOG_FILE" 2>&1 +``` + +## Performance Comparison + +| Mode | Speed | Storage | Dependencies | +|------|-------|---------|--------------| +| **Local** | Moderate | High (stores files) | getmail, filesystem | +| **Gmail Index** | Fast | Low (metadata only) | Gmail API, OAuth | + +## Related Files + +- [docker-compose.prod.yml](../docker-compose.prod.yml) - Production compose file with volume mounts +- [Dockerfile.prod.go](../Dockerfile.prod.go) - Builds both server and vault binaries +- [vault-cron-prod.sh](vault-cron-prod.sh) - Cron script for running vault +- [go-app/cmd/vault/README.md](../go-app/cmd/vault/README.md) - Vault program documentation diff --git a/scripts/VAULT_QUICKSTART.md b/scripts/VAULT_QUICKSTART.md new file mode 100644 index 00000000..46ccd841 --- /dev/null +++ b/scripts/VAULT_QUICKSTART.md @@ -0,0 +1,119 @@ +# Go Vault Quick Start + +## TL;DR - Run Vault in Docker Container + +### One-Time Setup + +```bash +# 1. Rebuild Docker image with vault binary +cd ~/src/cmc-sales +docker compose -f docker-compose.prod.yml build cmc-prod-go +docker compose -f docker-compose.prod.yml up -d cmc-prod-go + +# 2. Verify vault binary exists +docker exec -t cmc-prod-go ls -lh /root/vault + +# 3. Test run manually +docker exec -t cmc-prod-go ./vault --mode=local \ + --vaultdir=/var/www/vaultmsgs/new \ + --processeddir=/var/www/vaultmsgs/cur \ + --emaildir=/var/www/emails \ + --dbhost=cmc-prod-db \ + --dbuser=cmc \ + --dbpass="xVRQI&cA?7AU=hqJ!%au" \ + --dbname=cmc + +# 4. Copy cron script to server +# (assuming you're on your local machine) +scp scripts/vault-cron-prod.sh cmc@sales.cmctechnologies.com.au:~/scripts/ +ssh cmc@sales.cmctechnologies.com.au +chmod +x ~/scripts/vault-cron-prod.sh +mkdir -p ~/logs + +# 5. Add to crontab +crontab -e +# Add: */5 * * * * /home/cmc/scripts/vault-cron-prod.sh +``` + +### Manual Run (for testing) + +```bash +# Run vault once +docker exec -t cmc-prod-go ./vault --mode=local \ + --vaultdir=/var/www/vaultmsgs/new \ + --processeddir=/var/www/vaultmsgs/cur \ + --emaildir=/var/www/emails \ + --dbhost=cmc-prod-db \ + --dbuser=cmc \ + --dbpass="xVRQI&cA?7AU=hqJ!%au" \ + --dbname=cmc +``` + +### Monitor Logs + +```bash +# Watch the log file +tail -f ~/logs/vault-go.log + +# Check cron is running +crontab -l +ps aux | grep cron +``` + +## Command Variations + +### Local Mode (filesystem processing) +```bash +docker exec -t cmc-prod-go ./vault --mode=local \ + --vaultdir=/var/www/vaultmsgs/new \ + --processeddir=/var/www/vaultmsgs/cur \ + --emaildir=/var/www/emails \ + --dbhost=cmc-prod-db \ + --dbuser=cmc \ + --dbpass="xVRQI&cA?7AU=hqJ!%au" \ + --dbname=cmc +``` + +### Gmail Index Mode (requires OAuth setup) +```bash +docker exec -t cmc-prod-go ./vault --mode=index \ + --gmail-query="is:unread newer_than:1d" \ + --credentials=credentials.json \ + --token=token.json \ + --dbhost=cmc-prod-db \ + --dbuser=cmc \ + --dbpass="xVRQI&cA?7AU=hqJ!%au" \ + --dbname=cmc +``` + +## Troubleshooting + +### Vault binary not found +```bash +docker compose -f docker-compose.prod.yml build cmc-prod-go +docker compose -f docker-compose.prod.yml up -d cmc-prod-go +``` + +### Check what's in vaultmsgs/new +```bash +ls -la /home/cmc/files/vaultmsgs/new/ +``` + +### Database connection test +```bash +docker exec -it cmc-prod-go sh +nc -zv cmc-prod-db 3306 +``` + +### Check if directories are mounted +```bash +docker exec -it cmc-prod-go ls -la /var/www/vaultmsgs/ +``` + +## Files Changed + +- [Dockerfile.prod.go](../Dockerfile.prod.go) - Added vault binary build +- [docker-compose.prod.yml](../docker-compose.prod.yml) - Added volume mounts for vault directories +- [scripts/vault-cron-prod.sh](vault-cron-prod.sh) - Cron script + +See [VAULT_DEPLOYMENT.md](VAULT_DEPLOYMENT.md) for full documentation. diff --git a/scripts/vault-cron-prod.sh b/scripts/vault-cron-prod.sh new file mode 100755 index 00000000..5d4e1e79 --- /dev/null +++ b/scripts/vault-cron-prod.sh @@ -0,0 +1,38 @@ +#!/bin/bash +# Run Go vault in local mode inside Docker container +# Add to crontab: */5 * * * * /path/to/scripts/vault-cron-prod.sh + +CONTAINER_NAME="cmc-prod-go" +LOG_FILE="/home/cmc/logs/vault-go.log" + +# Ensure log directory exists +mkdir -p "$(dirname "$LOG_FILE")" + +# Check if container is running +if ! docker ps --format '{{.Names}}' | grep -q "^${CONTAINER_NAME}$"; then + echo "$(date): Error: Container $CONTAINER_NAME is not running" >> "$LOG_FILE" + exit 1 +fi + +# Run vault in local mode +echo "$(date): Starting vault processing..." >> "$LOG_FILE" + +docker exec -t "$CONTAINER_NAME" ./vault --mode=local \ + --vaultdir=/var/www/vaultmsgs/new \ + --processeddir=/var/www/vaultmsgs/cur \ + --emaildir=/var/www/emails \ + --dbhost=cmc-prod-db \ + --dbuser=cmc \ + --dbpass="xVRQI&cA?7AU=hqJ!%au" \ + --dbname=cmc \ + >> "$LOG_FILE" 2>&1 + +EXIT_CODE=$? + +if [ $EXIT_CODE -eq 0 ]; then + echo "$(date): Vault completed successfully" >> "$LOG_FILE" +else + echo "$(date): Vault failed with exit code $EXIT_CODE" >> "$LOG_FILE" +fi + +exit $EXIT_CODE