Merge branch 'prod' of code.springupsoftware.com:cmc/cmc-sales into prod
This commit is contained in:
commit
702186000d
|
|
@ -9,11 +9,13 @@ RUN go install github.com/sqlc-dev/sqlc/cmd/sqlc@latest
|
|||
RUN sqlc generate
|
||||
RUN go mod tidy
|
||||
RUN CGO_ENABLED=0 GOOS=linux go build -a -installsuffix cgo -o server cmd/server/main.go
|
||||
RUN CGO_ENABLED=0 GOOS=linux go build -a -installsuffix cgo -o vault cmd/vault/main.go
|
||||
|
||||
FROM alpine:latest
|
||||
RUN apk --no-cache add ca-certificates tzdata
|
||||
WORKDIR /root/
|
||||
COPY --from=builder /app/server .
|
||||
COPY --from=builder /app/vault .
|
||||
COPY go-app/templates ./templates
|
||||
COPY go-app/static ./static
|
||||
COPY go-app/.env.example .env
|
||||
|
|
|
|||
|
|
@ -81,6 +81,9 @@ services:
|
|||
- "8083:8082"
|
||||
volumes:
|
||||
- /home/cmc/files/pdf:/root/webroot/pdf:ro
|
||||
- /home/cmc/files/emails:/var/www/emails
|
||||
- /home/cmc/files/vault:/var/www/vault
|
||||
- /home/cmc/files/vaultmsgs:/var/www/vaultmsgs
|
||||
networks:
|
||||
- cmc-prod-network
|
||||
restart: unless-stopped
|
||||
|
|
|
|||
231
scripts/VAULT_DEPLOYMENT.md
Normal file
231
scripts/VAULT_DEPLOYMENT.md
Normal file
|
|
@ -0,0 +1,231 @@
|
|||
# Go Vault Deployment Guide
|
||||
|
||||
## Running the Go Vault in Docker Production
|
||||
|
||||
### Prerequisites
|
||||
|
||||
- The vault binary is built into the Docker image
|
||||
- Vault directories are mounted from `/home/cmc/files/vaultmsgs/`
|
||||
- Container has access to the database
|
||||
|
||||
### Step 1: Rebuild the Docker Image
|
||||
|
||||
The Dockerfile has been updated to include the vault binary. Rebuild the image:
|
||||
|
||||
```bash
|
||||
# On the production server
|
||||
cd ~/src/cmc-sales
|
||||
|
||||
# Rebuild the Go container
|
||||
docker compose -f docker-compose.prod.yml build cmc-prod-go
|
||||
|
||||
# Restart the container
|
||||
docker compose -f docker-compose.prod.yml up -d cmc-prod-go
|
||||
```
|
||||
|
||||
### Step 2: Verify Vault Binary Exists
|
||||
|
||||
```bash
|
||||
# Check if vault binary is in the container
|
||||
docker exec -t cmc-prod-go ls -lh /root/vault
|
||||
|
||||
# Test running vault (dry run)
|
||||
docker exec -t cmc-prod-go ./vault --mode=local \
|
||||
--vaultdir=/var/www/vaultmsgs/new \
|
||||
--processeddir=/var/www/vaultmsgs/cur \
|
||||
--emaildir=/var/www/emails \
|
||||
--dbhost=cmc-prod-db \
|
||||
--dbuser=cmc \
|
||||
--dbpass="xVRQI&cA?7AU=hqJ!%au" \
|
||||
--dbname=cmc
|
||||
```
|
||||
|
||||
### Step 3: Set Up Cron Job
|
||||
|
||||
```bash
|
||||
# Copy the cron script to the server
|
||||
scp scripts/vault-cron-prod.sh cmc@sales.cmctechnologies.com.au:~/scripts/
|
||||
|
||||
# On the server, make it executable
|
||||
chmod +x ~/scripts/vault-cron-prod.sh
|
||||
|
||||
# Create log directory
|
||||
mkdir -p ~/logs
|
||||
|
||||
# Test the script manually
|
||||
~/scripts/vault-cron-prod.sh
|
||||
|
||||
# Check the log
|
||||
tail -f ~/logs/vault-go.log
|
||||
```
|
||||
|
||||
### Step 4: Add to Crontab
|
||||
|
||||
```bash
|
||||
# Edit crontab
|
||||
crontab -e
|
||||
|
||||
# Add this line to run every 5 minutes
|
||||
*/5 * * * * /home/cmc/scripts/vault-cron-prod.sh
|
||||
```
|
||||
|
||||
### Step 5: Monitor
|
||||
|
||||
```bash
|
||||
# Watch logs in real-time
|
||||
tail -f ~/logs/vault-go.log
|
||||
|
||||
# Check cron execution
|
||||
grep vault /var/log/syslog
|
||||
|
||||
# List cron jobs
|
||||
crontab -l
|
||||
```
|
||||
|
||||
## Directory Structure
|
||||
|
||||
The following directories need to exist on the host:
|
||||
|
||||
```
|
||||
/home/cmc/files/
|
||||
├── vaultmsgs/
|
||||
│ ├── new/ # Incoming emails to process
|
||||
│ └── cur/ # Processed emails archive
|
||||
├── emails/ # Email storage
|
||||
└── vault/ # Additional vault data
|
||||
```
|
||||
|
||||
## Troubleshooting
|
||||
|
||||
### Vault binary not found
|
||||
|
||||
```bash
|
||||
# Rebuild the image
|
||||
docker compose -f docker-compose.prod.yml build cmc-prod-go
|
||||
docker compose -f docker-compose.prod.yml up -d cmc-prod-go
|
||||
```
|
||||
|
||||
### Permission errors
|
||||
|
||||
```bash
|
||||
# Check directory permissions on host
|
||||
ls -ld /home/cmc/files/vaultmsgs/new
|
||||
ls -ld /home/cmc/files/vaultmsgs/cur
|
||||
|
||||
# Should be readable/writable by the user running Docker
|
||||
```
|
||||
|
||||
### Database connection errors
|
||||
|
||||
```bash
|
||||
# Verify database container is running
|
||||
docker ps | grep cmc-prod-db
|
||||
|
||||
# Test database connection from Go container
|
||||
docker exec -it cmc-prod-go sh
|
||||
nc -zv cmc-prod-db 3306
|
||||
```
|
||||
|
||||
### No emails being processed
|
||||
|
||||
```bash
|
||||
# Check if there are emails to process
|
||||
ls -la /home/cmc/files/vaultmsgs/new/
|
||||
|
||||
# Check if getmail or other email fetcher is running
|
||||
ps aux | grep getmail
|
||||
```
|
||||
|
||||
## Switching from PHP to Go Vault
|
||||
|
||||
### Current PHP Setup
|
||||
|
||||
The PHP vault runs via:
|
||||
```bash
|
||||
/var/www/cmc-sales/cake/console/cake -app /var/www/cmc-sales/app vault
|
||||
```
|
||||
|
||||
### Migration Steps
|
||||
|
||||
1. **Run both in parallel** (testing phase):
|
||||
```bash
|
||||
# Keep PHP vault running
|
||||
*/5 * * * * /path/to/old/vault_cron.sh
|
||||
|
||||
# Add Go vault (different schedule)
|
||||
*/10 * * * * /home/cmc/scripts/vault-cron-prod.sh
|
||||
```
|
||||
|
||||
2. **Compare results**:
|
||||
- Check database for duplicate emails
|
||||
- Verify all business entities are linked correctly
|
||||
- Compare attachment handling
|
||||
|
||||
3. **Switch completely**:
|
||||
```bash
|
||||
# Disable PHP vault
|
||||
crontab -e # Comment out the PHP vault line
|
||||
|
||||
# Enable Go vault only
|
||||
*/5 * * * * /home/cmc/scripts/vault-cron-prod.sh
|
||||
```
|
||||
|
||||
## Advanced: Gmail Mode
|
||||
|
||||
For better performance, consider switching to Gmail API mode:
|
||||
|
||||
### Setup Gmail OAuth (one-time)
|
||||
|
||||
```bash
|
||||
# Copy credentials to the server
|
||||
scp credentials.json cmc@sales.cmctechnologies.com.au:~/
|
||||
|
||||
# Copy to container
|
||||
docker cp ~/credentials.json cmc-prod-go:/root/credentials.json
|
||||
|
||||
# Run initial authorization (interactive)
|
||||
docker exec -it cmc-prod-go ./vault --mode=index \
|
||||
--credentials=credentials.json \
|
||||
--token=token.json \
|
||||
--gmail-query="is:unread newer_than:1d" \
|
||||
--dbhost=cmc-prod-db \
|
||||
--dbuser=cmc \
|
||||
--dbpass="xVRQI&cA?7AU=hqJ!%au" \
|
||||
--dbname=cmc
|
||||
|
||||
# This will generate a URL - open in browser and authorize
|
||||
# Token will be saved in container
|
||||
|
||||
# Copy token back to host for backup
|
||||
docker cp cmc-prod-go:/root/token.json ~/token.json
|
||||
```
|
||||
|
||||
### Update Cron for Gmail Mode
|
||||
|
||||
Edit [scripts/vault-cron-prod.sh](scripts/vault-cron-prod.sh) and change:
|
||||
|
||||
```bash
|
||||
docker exec -t "$CONTAINER_NAME" ./vault --mode=index \
|
||||
--credentials=credentials.json \
|
||||
--token=token.json \
|
||||
--gmail-query="is:unread newer_than:1d" \
|
||||
--dbhost=cmc-prod-db \
|
||||
--dbuser=cmc \
|
||||
--dbpass="xVRQI&cA?7AU=hqJ!%au" \
|
||||
--dbname=cmc \
|
||||
>> "$LOG_FILE" 2>&1
|
||||
```
|
||||
|
||||
## Performance Comparison
|
||||
|
||||
| Mode | Speed | Storage | Dependencies |
|
||||
|------|-------|---------|--------------|
|
||||
| **Local** | Moderate | High (stores files) | getmail, filesystem |
|
||||
| **Gmail Index** | Fast | Low (metadata only) | Gmail API, OAuth |
|
||||
|
||||
## Related Files
|
||||
|
||||
- [docker-compose.prod.yml](../docker-compose.prod.yml) - Production compose file with volume mounts
|
||||
- [Dockerfile.prod.go](../Dockerfile.prod.go) - Builds both server and vault binaries
|
||||
- [vault-cron-prod.sh](vault-cron-prod.sh) - Cron script for running vault
|
||||
- [go-app/cmd/vault/README.md](../go-app/cmd/vault/README.md) - Vault program documentation
|
||||
119
scripts/VAULT_QUICKSTART.md
Normal file
119
scripts/VAULT_QUICKSTART.md
Normal file
|
|
@ -0,0 +1,119 @@
|
|||
# Go Vault Quick Start
|
||||
|
||||
## TL;DR - Run Vault in Docker Container
|
||||
|
||||
### One-Time Setup
|
||||
|
||||
```bash
|
||||
# 1. Rebuild Docker image with vault binary
|
||||
cd ~/src/cmc-sales
|
||||
docker compose -f docker-compose.prod.yml build cmc-prod-go
|
||||
docker compose -f docker-compose.prod.yml up -d cmc-prod-go
|
||||
|
||||
# 2. Verify vault binary exists
|
||||
docker exec -t cmc-prod-go ls -lh /root/vault
|
||||
|
||||
# 3. Test run manually
|
||||
docker exec -t cmc-prod-go ./vault --mode=local \
|
||||
--vaultdir=/var/www/vaultmsgs/new \
|
||||
--processeddir=/var/www/vaultmsgs/cur \
|
||||
--emaildir=/var/www/emails \
|
||||
--dbhost=cmc-prod-db \
|
||||
--dbuser=cmc \
|
||||
--dbpass="xVRQI&cA?7AU=hqJ!%au" \
|
||||
--dbname=cmc
|
||||
|
||||
# 4. Copy cron script to server
|
||||
# (assuming you're on your local machine)
|
||||
scp scripts/vault-cron-prod.sh cmc@sales.cmctechnologies.com.au:~/scripts/
|
||||
ssh cmc@sales.cmctechnologies.com.au
|
||||
chmod +x ~/scripts/vault-cron-prod.sh
|
||||
mkdir -p ~/logs
|
||||
|
||||
# 5. Add to crontab
|
||||
crontab -e
|
||||
# Add: */5 * * * * /home/cmc/scripts/vault-cron-prod.sh
|
||||
```
|
||||
|
||||
### Manual Run (for testing)
|
||||
|
||||
```bash
|
||||
# Run vault once
|
||||
docker exec -t cmc-prod-go ./vault --mode=local \
|
||||
--vaultdir=/var/www/vaultmsgs/new \
|
||||
--processeddir=/var/www/vaultmsgs/cur \
|
||||
--emaildir=/var/www/emails \
|
||||
--dbhost=cmc-prod-db \
|
||||
--dbuser=cmc \
|
||||
--dbpass="xVRQI&cA?7AU=hqJ!%au" \
|
||||
--dbname=cmc
|
||||
```
|
||||
|
||||
### Monitor Logs
|
||||
|
||||
```bash
|
||||
# Watch the log file
|
||||
tail -f ~/logs/vault-go.log
|
||||
|
||||
# Check cron is running
|
||||
crontab -l
|
||||
ps aux | grep cron
|
||||
```
|
||||
|
||||
## Command Variations
|
||||
|
||||
### Local Mode (filesystem processing)
|
||||
```bash
|
||||
docker exec -t cmc-prod-go ./vault --mode=local \
|
||||
--vaultdir=/var/www/vaultmsgs/new \
|
||||
--processeddir=/var/www/vaultmsgs/cur \
|
||||
--emaildir=/var/www/emails \
|
||||
--dbhost=cmc-prod-db \
|
||||
--dbuser=cmc \
|
||||
--dbpass="xVRQI&cA?7AU=hqJ!%au" \
|
||||
--dbname=cmc
|
||||
```
|
||||
|
||||
### Gmail Index Mode (requires OAuth setup)
|
||||
```bash
|
||||
docker exec -t cmc-prod-go ./vault --mode=index \
|
||||
--gmail-query="is:unread newer_than:1d" \
|
||||
--credentials=credentials.json \
|
||||
--token=token.json \
|
||||
--dbhost=cmc-prod-db \
|
||||
--dbuser=cmc \
|
||||
--dbpass="xVRQI&cA?7AU=hqJ!%au" \
|
||||
--dbname=cmc
|
||||
```
|
||||
|
||||
## Troubleshooting
|
||||
|
||||
### Vault binary not found
|
||||
```bash
|
||||
docker compose -f docker-compose.prod.yml build cmc-prod-go
|
||||
docker compose -f docker-compose.prod.yml up -d cmc-prod-go
|
||||
```
|
||||
|
||||
### Check what's in vaultmsgs/new
|
||||
```bash
|
||||
ls -la /home/cmc/files/vaultmsgs/new/
|
||||
```
|
||||
|
||||
### Database connection test
|
||||
```bash
|
||||
docker exec -it cmc-prod-go sh
|
||||
nc -zv cmc-prod-db 3306
|
||||
```
|
||||
|
||||
### Check if directories are mounted
|
||||
```bash
|
||||
docker exec -it cmc-prod-go ls -la /var/www/vaultmsgs/
|
||||
```
|
||||
|
||||
## Files Changed
|
||||
|
||||
- [Dockerfile.prod.go](../Dockerfile.prod.go) - Added vault binary build
|
||||
- [docker-compose.prod.yml](../docker-compose.prod.yml) - Added volume mounts for vault directories
|
||||
- [scripts/vault-cron-prod.sh](vault-cron-prod.sh) - Cron script
|
||||
|
||||
See [VAULT_DEPLOYMENT.md](VAULT_DEPLOYMENT.md) for full documentation.
|
||||
38
scripts/vault-cron-prod.sh
Executable file
38
scripts/vault-cron-prod.sh
Executable file
|
|
@ -0,0 +1,38 @@
|
|||
#!/bin/bash
|
||||
# Run Go vault in local mode inside Docker container
|
||||
# Add to crontab: */5 * * * * /path/to/scripts/vault-cron-prod.sh
|
||||
|
||||
CONTAINER_NAME="cmc-prod-go"
|
||||
LOG_FILE="/home/cmc/logs/vault-go.log"
|
||||
|
||||
# Ensure log directory exists
|
||||
mkdir -p "$(dirname "$LOG_FILE")"
|
||||
|
||||
# Check if container is running
|
||||
if ! docker ps --format '{{.Names}}' | grep -q "^${CONTAINER_NAME}$"; then
|
||||
echo "$(date): Error: Container $CONTAINER_NAME is not running" >> "$LOG_FILE"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Run vault in local mode
|
||||
echo "$(date): Starting vault processing..." >> "$LOG_FILE"
|
||||
|
||||
docker exec -t "$CONTAINER_NAME" ./vault --mode=local \
|
||||
--vaultdir=/var/www/vaultmsgs/new \
|
||||
--processeddir=/var/www/vaultmsgs/cur \
|
||||
--emaildir=/var/www/emails \
|
||||
--dbhost=cmc-prod-db \
|
||||
--dbuser=cmc \
|
||||
--dbpass="xVRQI&cA?7AU=hqJ!%au" \
|
||||
--dbname=cmc \
|
||||
>> "$LOG_FILE" 2>&1
|
||||
|
||||
EXIT_CODE=$?
|
||||
|
||||
if [ $EXIT_CODE -eq 0 ]; then
|
||||
echo "$(date): Vault completed successfully" >> "$LOG_FILE"
|
||||
else
|
||||
echo "$(date): Vault failed with exit code $EXIT_CODE" >> "$LOG_FILE"
|
||||
fi
|
||||
|
||||
exit $EXIT_CODE
|
||||
Loading…
Reference in a new issue