cmc-sales/go-app/cmd/vault/main.go

package main

import (
	"bytes"
	"database/sql"
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"path/filepath"
	"regexp"
	"strings"
	"time"

	_ "github.com/go-sql-driver/mysql"
	"github.com/google/uuid"
	"github.com/jhillyerd/enmime"
)

type Config struct {
	EmailDir      string
	VaultDir      string
	ProcessedDir  string
	DBHost        string
	DBUser        string
	DBPassword    string
	DBName        string
}

type EmailProcessor struct {
	db           *sql.DB
	config       Config
	enquiryMap   map[string]int
	invoiceMap   map[string]int
	poMap        map[string]int
	userMap      map[string]int
	jobMap       map[string]int
}

type Attachment struct {
	Type          string
	Name          string
	Filename      string
	Size          int64
	IsMessageBody int
}

func main() {
	var config Config
	flag.StringVar(&config.EmailDir, "emaildir", "/var/www/emails", "Email storage directory")
	flag.StringVar(&config.VaultDir, "vaultdir", "/var/www/vaultmsgs/new", "Vault messages directory")
	flag.StringVar(&config.ProcessedDir, "processeddir", "/var/www/vaultmsgs/cur", "Processed messages directory")
	flag.StringVar(&config.DBHost, "dbhost", "127.0.0.1", "Database host")
	flag.StringVar(&config.DBUser, "dbuser", "cmc", "Database user")
	flag.StringVar(&config.DBPassword, "dbpass", "xVRQI&cA?7AU=hqJ!%au", "Database password")
	flag.StringVar(&config.DBName, "dbname", "cmc", "Database name")
	flag.Parse()

	dsn := fmt.Sprintf("%s:%s@tcp(%s)/%s?parseTime=true",
		config.DBUser, config.DBPassword, config.DBHost, config.DBName)
	
	db, err := sql.Open("mysql", dsn)
	if err != nil {
		log.Fatal("Failed to connect to database:", err)
	}
	defer db.Close()

	processor := &EmailProcessor{
		db:     db,
		config: config,
	}

	if err := processor.loadMaps(); err != nil {
		log.Fatal("Failed to load maps:", err)
	}

	if err := processor.processEmails(); err != nil {
		log.Fatal("Failed to process emails:", err)
	}
}

func (p *EmailProcessor) loadMaps() error {
	p.enquiryMap = make(map[string]int)
	p.invoiceMap = make(map[string]int)
	p.poMap = make(map[string]int)
	p.userMap = make(map[string]int)
	p.jobMap = make(map[string]int)

	// Load enquiries
	rows, err := p.db.Query("SELECT id, title FROM enquiries")
	if err != nil {
		return err
	}
	defer rows.Close()
	for rows.Next() {
		var id int
		var title string
		if err := rows.Scan(&id, &title); err != nil {
			return err
		}
		p.enquiryMap[title] = id
	}

	// Load invoices
	rows, err = p.db.Query("SELECT id, title FROM invoices")
	if err != nil {
		return err
	}
	defer rows.Close()
	for rows.Next() {
		var id int
		var title string
		if err := rows.Scan(&id, &title); err != nil {
			return err
		}
		p.invoiceMap[title] = id
	}

	// Load purchase orders
	rows, err = p.db.Query("SELECT id, title FROM purchase_orders")
	if err != nil {
		return err
	}
	defer rows.Close()
	for rows.Next() {
		var id int
		var title string
		if err := rows.Scan(&id, &title); err != nil {
			return err
		}
		p.poMap[title] = id
	}

	// Load users
	rows, err = p.db.Query("SELECT id, email FROM users")
	if err != nil {
		return err
	}
	defer rows.Close()
	for rows.Next() {
		var id int
		var email string
		if err := rows.Scan(&id, &email); err != nil {
			return err
		}
		p.userMap[strings.ToLower(email)] = id
	}

	// Load jobs
	rows, err = p.db.Query("SELECT id, title FROM jobs")
	if err != nil {
		return err
	}
	defer rows.Close()
	for rows.Next() {
		var id int
		var title string
		if err := rows.Scan(&id, &title); err != nil {
			return err
		}
		p.jobMap[title] = id
	}

	return nil
}

func (p *EmailProcessor) processEmails() error {
	files, err := ioutil.ReadDir(p.config.VaultDir)
	if err != nil {
		return err
	}

	for _, file := range files {
		if file.IsDir() || file.Name() == "." || file.Name() == ".." {
			continue
		}

		fmt.Printf("Handling %s\n", file.Name())
		if err := p.processEmail(file.Name()); err != nil {
			log.Printf("Error processing %s: %v\n", file.Name(), err)
		}
	}

	return nil
}

func (p *EmailProcessor) processEmail(filename string) error {
	emailPath := filepath.Join(p.config.VaultDir, filename)
	content, err := ioutil.ReadFile(emailPath)
	if err != nil {
		return err
	}

	if len(content) == 0 {
		fmt.Println("No content found. Ignoring this email")
		return p.moveEmail(filename)
	}

	// Parse email with enmime
	env, err := enmime.ReadEnvelope(bytes.NewReader(content))
	if err != nil {
		return err
	}

	// Get recipients
	toRecipients := p.parseEnmimeAddresses(env.GetHeader("To"))
	fromRecipients := p.parseEnmimeAddresses(env.GetHeader("From"))
	ccRecipients := p.parseEnmimeAddresses(env.GetHeader("Cc"))

	// Check if we should save this email
	saveThis := false
	fromKnownUser := false

	for _, email := range toRecipients {
		if p.userExists(email) {
			saveThis = true
		}
	}

	for _, email := range fromRecipients {
		if p.userExists(email) {
			saveThis = true
			fromKnownUser = true
		}
	}

	for _, email := range ccRecipients {
		if p.userExists(email) {
			saveThis = true
		}
	}

	subject := env.GetHeader("Subject")
	if subject == "" {
		fmt.Println("No subject found. Ignoring this email")
		return p.moveEmail(filename)
	}

	// Check for identifiers in subject
	foundEnquiries := p.checkIdentifier(subject, p.enquiryMap, "enquiry")
	foundInvoices := p.checkIdentifier(subject, p.invoiceMap, "invoice")
	foundPOs := p.checkIdentifier(subject, p.poMap, "purchaseorder")
	foundJobs := p.checkIdentifier(subject, p.jobMap, "job")

	foundIdent := len(foundEnquiries) > 0 || len(foundInvoices) > 0 || 
		len(foundPOs) > 0 || len(foundJobs) > 0

	if fromKnownUser || saveThis || foundIdent {
		// Process and save the email
		unixTime := time.Now().Unix()
		if date, err := env.Date(); err == nil {
			unixTime = date.Unix()
		}

		// Get recipient user IDs
		recipientIDs := make(map[string][]int)
		recipientIDs["to"] = p.getUserIDs(toRecipients)
		recipientIDs["from"] = p.getUserIDs(fromRecipients)
		recipientIDs["cc"] = p.getUserIDs(ccRecipients)

		if len(recipientIDs["from"]) == 0 {
			fmt.Println("Email has no From Recipient ID. Ignoring this email")
			return p.moveEmail(filename)
		}

		fmt.Println("---------START MESSAGE -----------------")
		fmt.Printf("Subject: %s\n", subject)

		// Extract attachments using enmime
		relativePath := p.getAttachmentDirectory(unixTime)
		attachments := p.extractEnmimeAttachments(env, relativePath)

		// Save email to database
		if err := p.saveEmail(filename, subject, unixTime, recipientIDs, attachments,
			foundEnquiries, foundInvoices, foundPOs, foundJobs); err != nil {
			return err
		}

		fmt.Println("--------END MESSAGE ------")
	} else {
		fmt.Printf("Email will not be saved. Subject: %s\n", subject)
	}

	return p.moveEmail(filename)
}

func (p *EmailProcessor) parseEnmimeAddresses(header string) []string {
	var emails []string
	if header == "" {
		return emails
	}

	addresses, err := enmime.ParseAddressList(header)
	if err != nil {
		return emails
	}

	for _, addr := range addresses {
		emails = append(emails, strings.ToLower(addr.Address))
	}

	return emails
}

func (p *EmailProcessor) userExists(email string) bool {
	_, exists := p.userMap[strings.ToLower(email)]
	return exists
}

func (p *EmailProcessor) getUserIDs(emails []string) []int {
	var ids []int
	for _, email := range emails {
		if id, exists := p.userMap[strings.ToLower(email)]; exists {
			ids = append(ids, id)
		} else {
			// Create new user
			newID := p.createUser(email)
			if newID > 0 {
				ids = append(ids, newID)
				p.userMap[strings.ToLower(email)] = newID
			}
		}
	}
	return ids
}

func (p *EmailProcessor) createUser(email string) int {
	fmt.Printf("Making a new User for: '%s'\n", email)
	
	result, err := p.db.Exec(
		"INSERT INTO users (type, email, by_vault, created, modified) VALUES (?, ?, ?, NOW(), NOW())",
		"contact", strings.ToLower(email), 1)
	
	if err != nil {
		fmt.Printf("Serious Error: Unable to create user for email '%s': %v\n", email, err)
		return 0
	}

	id, err := result.LastInsertId()
	if err != nil {
		return 0
	}

	fmt.Printf("New User '%s' Added with ID: %d\n", email, id)
	return int(id)
}

func (p *EmailProcessor) checkIdentifier(subject string, identMap map[string]int, identType string) []int {
	var results []int
	var re *regexp.Regexp

	switch identType {
	case "enquiry":
		re = regexp.MustCompile(`CMC\d+([NVQWSOT]|ACT|NT)E\d+-\d+`)
	case "invoice":
		re = regexp.MustCompile(`CMCIN\d+`)
	case "purchaseorder":
		re = regexp.MustCompile(`CMCPO\d+`)
	case "job":
		re = regexp.MustCompile(`(JAN|FEB|MAR|APR|MAY|JUN|JUL|AUG|SEP|OCT|NOV|DEC)\d+(N|V|W|S|T|NT|ACT|Q|O)J\d+`)
	}

	if re != nil {
		matches := re.FindAllString(subject, -1)
		for _, match := range matches {
			if id, exists := identMap[match]; exists {
				results = append(results, id)
			}
		}
	}

	return results
}

func (p *EmailProcessor) getAttachmentDirectory(unixTime int64) string {
	t := time.Unix(unixTime, 0)
	monthYear := t.Format("01-2006")
	path := filepath.Join(p.config.EmailDir, monthYear)

	if err := os.MkdirAll(path, 0755); err != nil {
		log.Printf("Failed to create directory %s: %v", path, err)
	}

	return monthYear
}

func (p *EmailProcessor) extractEnmimeAttachments(env *enmime.Envelope, relativePath string) []Attachment {
	var attachments []Attachment
	outputDir := filepath.Join(p.config.EmailDir, relativePath)
	uuid := uuid.New().String()

	// Ensure output directory exists
	if err := os.MkdirAll(outputDir, 0755); err != nil {
		log.Printf("Failed to create output directory: %v", err)
		return attachments
	}

	biggestHTMLSize := int64(0)
	biggestHTMLIdx := -1
	biggestPlainSize := int64(0)
	biggestPlainIdx := -1

	// Process HTML part if exists
	if env.HTML != "" {
		htmlData := []byte(env.HTML)
		fileName := "texthtml"
		newFileName := uuid + "-" + fileName
		filePath := filepath.Join(outputDir, newFileName)
		
		if err := ioutil.WriteFile(filePath, htmlData, 0644); err == nil {
			att := Attachment{
				Type:          "text/html",
				Name:          filepath.Join(relativePath, newFileName),
				Filename:      fileName,
				Size:          int64(len(htmlData)),
				IsMessageBody: 0,
			}
			attachments = append(attachments, att)
			biggestHTMLIdx = len(attachments) - 1
			biggestHTMLSize = int64(len(htmlData))
		}
	}

	// Process plain text part if exists
	if env.Text != "" {
		textData := []byte(env.Text)
		fileName := "textplain"
		newFileName := uuid + "-" + fileName
		filePath := filepath.Join(outputDir, newFileName)
		
		if err := ioutil.WriteFile(filePath, textData, 0644); err == nil {
			att := Attachment{
				Type:          "text/plain",
				Name:          filepath.Join(relativePath, newFileName),
				Filename:      fileName,
				Size:          int64(len(textData)),
				IsMessageBody: 0,
			}
			attachments = append(attachments, att)
			biggestPlainIdx = len(attachments) - 1
			biggestPlainSize = int64(len(textData))
		}
	}

	// Process file attachments
	for _, part := range env.Attachments {
		fileName := part.FileName
		if fileName == "" {
			fileName = "attachment"
		}
		
		newFileName := uuid + "-" + fileName
		filePath := filepath.Join(outputDir, newFileName)
		
		if err := ioutil.WriteFile(filePath, part.Content, 0644); err != nil {
			log.Printf("Failed to save attachment %s: %v", fileName, err)
			continue
		}

		att := Attachment{
			Type:          part.ContentType,
			Name:          filepath.Join(relativePath, newFileName),
			Filename:      fileName,
			Size:          int64(len(part.Content)),
			IsMessageBody: 0,
		}

		attachments = append(attachments, att)
		idx := len(attachments) - 1

		// Track largest HTML and plain text attachments
		if strings.HasPrefix(part.ContentType, "text/html") && int64(len(part.Content)) > biggestHTMLSize {
			biggestHTMLSize = int64(len(part.Content))
			biggestHTMLIdx = idx
		} else if strings.HasPrefix(part.ContentType, "text/plain") && int64(len(part.Content)) > biggestPlainSize {
			biggestPlainSize = int64(len(part.Content))
			biggestPlainIdx = idx
		}
	}

	// Process inline parts
	for _, part := range env.Inlines {
		fileName := part.FileName
		if fileName == "" {
			fileName = "inline"
		}
		
		newFileName := uuid + "-" + fileName
		filePath := filepath.Join(outputDir, newFileName)
		
		if err := ioutil.WriteFile(filePath, part.Content, 0644); err != nil {
			log.Printf("Failed to save inline part %s: %v", fileName, err)
			continue
		}

		att := Attachment{
			Type:          part.ContentType,
			Name:          filepath.Join(relativePath, newFileName),
			Filename:      fileName,
			Size:          int64(len(part.Content)),
			IsMessageBody: 0,
		}

		attachments = append(attachments, att)
		idx := len(attachments) - 1

		// Track largest HTML and plain text attachments
		if strings.HasPrefix(part.ContentType, "text/html") && int64(len(part.Content)) > biggestHTMLSize {
			biggestHTMLSize = int64(len(part.Content))
			biggestHTMLIdx = idx
		} else if strings.HasPrefix(part.ContentType, "text/plain") && int64(len(part.Content)) > biggestPlainSize {
			biggestPlainSize = int64(len(part.Content))
			biggestPlainIdx = idx
		}
	}

	// Mark the message body
	if biggestHTMLIdx >= 0 {
		attachments[biggestHTMLIdx].IsMessageBody = 1
	} else if biggestPlainIdx >= 0 {
		attachments[biggestPlainIdx].IsMessageBody = 1
	}

	return attachments
}

func (p *EmailProcessor) saveEmail(filename, subject string, unixTime int64, 
	recipientIDs map[string][]int, attachments []Attachment,
	foundEnquiries, foundInvoices, foundPOs, foundJobs []int) error {

	tx, err := p.db.Begin()
	if err != nil {
		return err
	}
	defer tx.Rollback()

	// Insert email
	result, err := tx.Exec(
		"INSERT INTO emails (user_id, udate, created, subject, filename) VALUES (?, ?, NOW(), ?, ?)",
		recipientIDs["from"][0], unixTime, subject, filename)
	
	if err != nil {
		return err
	}

	emailID, err := result.LastInsertId()
	if err != nil {
		return err
	}

	// Insert recipients
	for recipType, userIDs := range recipientIDs {
		for _, userID := range userIDs {
			if recipType == "from" {
				continue // From is already stored in emails.user_id
			}
			_, err = tx.Exec(
				"INSERT INTO email_recipients (email_id, user_id, type) VALUES (?, ?, ?)",
				emailID, userID, recipType)
			if err != nil {
				return err
			}
		}
	}

	// Insert attachments
	for _, att := range attachments {
		_, err = tx.Exec(
			"INSERT INTO email_attachments (email_id, name, type, size, filename, is_message_body, created) VALUES (?, ?, ?, ?, ?, ?, NOW())",
			emailID, att.Name, att.Type, att.Size, att.Filename, att.IsMessageBody)
		if err != nil {
			return err
		}
	}

	// Insert associations
	for _, jobID := range foundJobs {
		_, err = tx.Exec("INSERT INTO emails_jobs (email_id, job_id) VALUES (?, ?)", emailID, jobID)
		if err != nil {
			return err
		}
	}

	for _, poID := range foundPOs {
		_, err = tx.Exec("INSERT INTO emails_purchase_orders (email_id, purchase_order_id) VALUES (?, ?)", emailID, poID)
		if err != nil {
			return err
		}
	}

	for _, enqID := range foundEnquiries {
		_, err = tx.Exec("INSERT INTO emails_enquiries (email_id, enquiry_id) VALUES (?, ?)", emailID, enqID)
		if err != nil {
			return err
		}
	}

	for _, invID := range foundInvoices {
		_, err = tx.Exec("INSERT INTO emails_invoices (email_id, invoice_id) VALUES (?, ?)", emailID, invID)
		if err != nil {
			return err
		}
	}

	if err := tx.Commit(); err != nil {
		return err
	}

	fmt.Println("Success. We made an email")
	return nil
}

func (p *EmailProcessor) moveEmail(filename string) error {
	oldPath := filepath.Join(p.config.VaultDir, filename)
	newPath := filepath.Join(p.config.ProcessedDir, filename+":S")

	if err := os.Rename(oldPath, newPath); err != nil {
		fmt.Printf("Unable to move %s to %s: %v\n", oldPath, newPath, err)
		return err
	}

	return nil
}