Merge branch 'prod' of code.springupsoftware.com:cmc/cmc-sales into prod

Dockerfile.prod.go

@@ -9,11 +9,13 @@ RUN go install github.com/sqlc-dev/sqlc/cmd/sqlc@latest
 RUN sqlc generate
 RUN go mod tidy
 RUN CGO_ENABLED=0 GOOS=linux go build -a -installsuffix cgo -o server cmd/server/main.go
+RUN CGO_ENABLED=0 GOOS=linux go build -a -installsuffix cgo -o vault cmd/vault/main.go
 
 FROM alpine:latest
 RUN apk --no-cache add ca-certificates tzdata
 WORKDIR /root/
 COPY --from=builder /app/server .
+COPY --from=builder /app/vault .
 COPY go-app/templates ./templates
 COPY go-app/static ./static
 COPY go-app/.env.example .env

docker-compose.prod.yml

@@ -81,6 +81,9 @@ services:
       - "8083:8082"
     volumes:
       - /home/cmc/files/pdf:/root/webroot/pdf:ro
+      - /home/cmc/files/emails:/var/www/emails
+      - /home/cmc/files/vault:/var/www/vault
+      - /home/cmc/files/vaultmsgs:/var/www/vaultmsgs
     networks:
       - cmc-prod-network
     restart: unless-stopped

scripts/VAULT_DEPLOYMENT.md

@@ -0,0 +1,231 @@
+# Go Vault Deployment Guide
+
+## Running the Go Vault in Docker Production
+
+### Prerequisites
+
+- The vault binary is built into the Docker image
+- Vault directories are mounted from `/home/cmc/files/vaultmsgs/`
+- Container has access to the database
+
+### Step 1: Rebuild the Docker Image
+
+The Dockerfile has been updated to include the vault binary. Rebuild the image:
+
+```bash
+# On the production server
+cd ~/src/cmc-sales
+
+# Rebuild the Go container
+docker compose -f docker-compose.prod.yml build cmc-prod-go
+
+# Restart the container
+docker compose -f docker-compose.prod.yml up -d cmc-prod-go
+```
+
+### Step 2: Verify Vault Binary Exists
+
+```bash
+# Check if vault binary is in the container
+docker exec -t cmc-prod-go ls -lh /root/vault
+
+# Test running vault (dry run)
+docker exec -t cmc-prod-go ./vault --mode=local \
+  --vaultdir=/var/www/vaultmsgs/new \
+  --processeddir=/var/www/vaultmsgs/cur \
+  --emaildir=/var/www/emails \
+  --dbhost=cmc-prod-db \
+  --dbuser=cmc \
+  --dbpass="xVRQI&cA?7AU=hqJ!%au" \
+  --dbname=cmc
+```
+
+### Step 3: Set Up Cron Job
+
+```bash
+# Copy the cron script to the server
+scp scripts/vault-cron-prod.sh cmc@sales.cmctechnologies.com.au:~/scripts/
+
+# On the server, make it executable
+chmod +x ~/scripts/vault-cron-prod.sh
+
+# Create log directory
+mkdir -p ~/logs
+
+# Test the script manually
+~/scripts/vault-cron-prod.sh
+
+# Check the log
+tail -f ~/logs/vault-go.log
+```
+
+### Step 4: Add to Crontab
+
+```bash
+# Edit crontab
+crontab -e
+
+# Add this line to run every 5 minutes
+*/5 * * * * /home/cmc/scripts/vault-cron-prod.sh
+```
+
+### Step 5: Monitor
+
+```bash
+# Watch logs in real-time
+tail -f ~/logs/vault-go.log
+
+# Check cron execution
+grep vault /var/log/syslog
+
+# List cron jobs
+crontab -l
+```
+
+## Directory Structure
+
+The following directories need to exist on the host:
+
+```
+/home/cmc/files/
+├── vaultmsgs/
+│   ├── new/       # Incoming emails to process
+│   └── cur/       # Processed emails archive
+├── emails/        # Email storage
+└── vault/         # Additional vault data
+```
+
+## Troubleshooting
+
+### Vault binary not found
+
+```bash
+# Rebuild the image
+docker compose -f docker-compose.prod.yml build cmc-prod-go
+docker compose -f docker-compose.prod.yml up -d cmc-prod-go
+```
+
+### Permission errors
+
+```bash
+# Check directory permissions on host
+ls -ld /home/cmc/files/vaultmsgs/new
+ls -ld /home/cmc/files/vaultmsgs/cur
+
+# Should be readable/writable by the user running Docker
+```
+
+### Database connection errors
+
+```bash
+# Verify database container is running
+docker ps | grep cmc-prod-db
+
+# Test database connection from Go container
+docker exec -it cmc-prod-go sh
+nc -zv cmc-prod-db 3306
+```
+
+### No emails being processed
+
+```bash
+# Check if there are emails to process
+ls -la /home/cmc/files/vaultmsgs/new/
+
+# Check if getmail or other email fetcher is running
+ps aux | grep getmail
+```
+
+## Switching from PHP to Go Vault
+
+### Current PHP Setup
+
+The PHP vault runs via:
+```bash
+/var/www/cmc-sales/cake/console/cake -app /var/www/cmc-sales/app vault
+```
+
+### Migration Steps
+
+1. **Run both in parallel** (testing phase):
+   ```bash
+   # Keep PHP vault running
+   */5 * * * * /path/to/old/vault_cron.sh
+
+   # Add Go vault (different schedule)
+   */10 * * * * /home/cmc/scripts/vault-cron-prod.sh
+   ```
+
+2. **Compare results**:
+   - Check database for duplicate emails
+   - Verify all business entities are linked correctly
+   - Compare attachment handling
+
+3. **Switch completely**:
+   ```bash
+   # Disable PHP vault
+   crontab -e  # Comment out the PHP vault line
+
+   # Enable Go vault only
+   */5 * * * * /home/cmc/scripts/vault-cron-prod.sh
+   ```
+
+## Advanced: Gmail Mode
+
+For better performance, consider switching to Gmail API mode:
+
+### Setup Gmail OAuth (one-time)
+
+```bash
+# Copy credentials to the server
+scp credentials.json cmc@sales.cmctechnologies.com.au:~/
+
+# Copy to container
+docker cp ~/credentials.json cmc-prod-go:/root/credentials.json
+
+# Run initial authorization (interactive)
+docker exec -it cmc-prod-go ./vault --mode=index \
+  --credentials=credentials.json \
+  --token=token.json \
+  --gmail-query="is:unread newer_than:1d" \
+  --dbhost=cmc-prod-db \
+  --dbuser=cmc \
+  --dbpass="xVRQI&cA?7AU=hqJ!%au" \
+  --dbname=cmc
+
+# This will generate a URL - open in browser and authorize
+# Token will be saved in container
+
+# Copy token back to host for backup
+docker cp cmc-prod-go:/root/token.json ~/token.json
+```
+
+### Update Cron for Gmail Mode
+
+Edit [scripts/vault-cron-prod.sh](scripts/vault-cron-prod.sh) and change:
+
+```bash
+docker exec -t "$CONTAINER_NAME" ./vault --mode=index \
+  --credentials=credentials.json \
+  --token=token.json \
+  --gmail-query="is:unread newer_than:1d" \
+  --dbhost=cmc-prod-db \
+  --dbuser=cmc \
+  --dbpass="xVRQI&cA?7AU=hqJ!%au" \
+  --dbname=cmc \
+  >> "$LOG_FILE" 2>&1
+```
+
+## Performance Comparison
+
+| Mode | Speed | Storage | Dependencies |
+|------|-------|---------|--------------|
+| **Local** | Moderate | High (stores files) | getmail, filesystem |
+| **Gmail Index** | Fast | Low (metadata only) | Gmail API, OAuth |
+
+## Related Files
+
+- [docker-compose.prod.yml](../docker-compose.prod.yml) - Production compose file with volume mounts
+- [Dockerfile.prod.go](../Dockerfile.prod.go) - Builds both server and vault binaries
+- [vault-cron-prod.sh](vault-cron-prod.sh) - Cron script for running vault
+- [go-app/cmd/vault/README.md](../go-app/cmd/vault/README.md) - Vault program documentation

scripts/VAULT_QUICKSTART.md

@@ -0,0 +1,119 @@
+# Go Vault Quick Start
+
+## TL;DR - Run Vault in Docker Container
+
+### One-Time Setup
+
+```bash
+# 1. Rebuild Docker image with vault binary
+cd ~/src/cmc-sales
+docker compose -f docker-compose.prod.yml build cmc-prod-go
+docker compose -f docker-compose.prod.yml up -d cmc-prod-go
+
+# 2. Verify vault binary exists
+docker exec -t cmc-prod-go ls -lh /root/vault
+
+# 3. Test run manually
+docker exec -t cmc-prod-go ./vault --mode=local \
+  --vaultdir=/var/www/vaultmsgs/new \
+  --processeddir=/var/www/vaultmsgs/cur \
+  --emaildir=/var/www/emails \
+  --dbhost=cmc-prod-db \
+  --dbuser=cmc \
+  --dbpass="xVRQI&cA?7AU=hqJ!%au" \
+  --dbname=cmc
+
+# 4. Copy cron script to server
+# (assuming you're on your local machine)
+scp scripts/vault-cron-prod.sh cmc@sales.cmctechnologies.com.au:~/scripts/
+ssh cmc@sales.cmctechnologies.com.au
+chmod +x ~/scripts/vault-cron-prod.sh
+mkdir -p ~/logs
+
+# 5. Add to crontab
+crontab -e
+# Add: */5 * * * * /home/cmc/scripts/vault-cron-prod.sh
+```
+
+### Manual Run (for testing)
+
+```bash
+# Run vault once
+docker exec -t cmc-prod-go ./vault --mode=local \
+  --vaultdir=/var/www/vaultmsgs/new \
+  --processeddir=/var/www/vaultmsgs/cur \
+  --emaildir=/var/www/emails \
+  --dbhost=cmc-prod-db \
+  --dbuser=cmc \
+  --dbpass="xVRQI&cA?7AU=hqJ!%au" \
+  --dbname=cmc
+```
+
+### Monitor Logs
+
+```bash
+# Watch the log file
+tail -f ~/logs/vault-go.log
+
+# Check cron is running
+crontab -l
+ps aux | grep cron
+```
+
+## Command Variations
+
+### Local Mode (filesystem processing)
+```bash
+docker exec -t cmc-prod-go ./vault --mode=local \
+  --vaultdir=/var/www/vaultmsgs/new \
+  --processeddir=/var/www/vaultmsgs/cur \
+  --emaildir=/var/www/emails \
+  --dbhost=cmc-prod-db \
+  --dbuser=cmc \
+  --dbpass="xVRQI&cA?7AU=hqJ!%au" \
+  --dbname=cmc
+```
+
+### Gmail Index Mode (requires OAuth setup)
+```bash
+docker exec -t cmc-prod-go ./vault --mode=index \
+  --gmail-query="is:unread newer_than:1d" \
+  --credentials=credentials.json \
+  --token=token.json \
+  --dbhost=cmc-prod-db \
+  --dbuser=cmc \
+  --dbpass="xVRQI&cA?7AU=hqJ!%au" \
+  --dbname=cmc
+```
+
+## Troubleshooting
+
+### Vault binary not found
+```bash
+docker compose -f docker-compose.prod.yml build cmc-prod-go
+docker compose -f docker-compose.prod.yml up -d cmc-prod-go
+```
+
+### Check what's in vaultmsgs/new
+```bash
+ls -la /home/cmc/files/vaultmsgs/new/
+```
+
+### Database connection test
+```bash
+docker exec -it cmc-prod-go sh
+nc -zv cmc-prod-db 3306
+```
+
+### Check if directories are mounted
+```bash
+docker exec -it cmc-prod-go ls -la /var/www/vaultmsgs/
+```
+
+## Files Changed
+
+- [Dockerfile.prod.go](../Dockerfile.prod.go) - Added vault binary build
+- [docker-compose.prod.yml](../docker-compose.prod.yml) - Added volume mounts for vault directories
+- [scripts/vault-cron-prod.sh](vault-cron-prod.sh) - Cron script
+
+See [VAULT_DEPLOYMENT.md](VAULT_DEPLOYMENT.md) for full documentation.

scripts/vault-cron-prod.sh

@@ -0,0 +1,38 @@
+#!/bin/bash
+# Run Go vault in local mode inside Docker container
+# Add to crontab: */5 * * * * /path/to/scripts/vault-cron-prod.sh
+
+CONTAINER_NAME="cmc-prod-go"
+LOG_FILE="/home/cmc/logs/vault-go.log"
+
+# Ensure log directory exists
+mkdir -p "$(dirname "$LOG_FILE")"
+
+# Check if container is running
+if ! docker ps --format '{{.Names}}' | grep -q "^${CONTAINER_NAME}$"; then
+  echo "$(date): Error: Container $CONTAINER_NAME is not running" >> "$LOG_FILE"
+  exit 1
+fi
+
+# Run vault in local mode
+echo "$(date): Starting vault processing..." >> "$LOG_FILE"
+
+docker exec -t "$CONTAINER_NAME" ./vault --mode=local \
+  --vaultdir=/var/www/vaultmsgs/new \
+  --processeddir=/var/www/vaultmsgs/cur \
+  --emaildir=/var/www/emails \
+  --dbhost=cmc-prod-db \
+  --dbuser=cmc \
+  --dbpass="xVRQI&cA?7AU=hqJ!%au" \
+  --dbname=cmc \
+  >> "$LOG_FILE" 2>&1
+
+EXIT_CODE=$?
+
+if [ $EXIT_CODE -eq 0 ]; then
+  echo "$(date): Vault completed successfully" >> "$LOG_FILE"
+else
+  echo "$(date): Vault failed with exit code $EXIT_CODE" >> "$LOG_FILE"
+fi
+
+exit $EXIT_CODE