Add go rewrite of vault
This commit is contained in:
parent
bc7c99007b
commit
f6eef99d47
|
|
@ -19,11 +19,24 @@ sqlc: ## Generate Go code from SQL queries
|
||||||
.PHONY: build
|
.PHONY: build
|
||||||
build: sqlc ## Build the application
|
build: sqlc ## Build the application
|
||||||
go build -o bin/server cmd/server/main.go
|
go build -o bin/server cmd/server/main.go
|
||||||
|
go build -o bin/vault cmd/vault/main.go
|
||||||
|
|
||||||
|
.PHONY: build-server
|
||||||
|
build-server: sqlc ## Build only the server
|
||||||
|
go build -o bin/server cmd/server/main.go
|
||||||
|
|
||||||
|
.PHONY: build-vault
|
||||||
|
build-vault: ## Build only the vault command
|
||||||
|
go build -o bin/vault cmd/vault/main.go
|
||||||
|
|
||||||
.PHONY: run
|
.PHONY: run
|
||||||
run: ## Run the application
|
run: ## Run the application
|
||||||
go run cmd/server/main.go
|
go run cmd/server/main.go
|
||||||
|
|
||||||
|
.PHONY: run-vault
|
||||||
|
run-vault: ## Run the vault command
|
||||||
|
go run cmd/vault/main.go
|
||||||
|
|
||||||
.PHONY: dev
|
.PHONY: dev
|
||||||
dev: sqlc ## Run the application with hot reload (requires air)
|
dev: sqlc ## Run the application with hot reload (requires air)
|
||||||
air
|
air
|
||||||
|
|
|
||||||
57
go-app/cmd/vault/README.md
Normal file
57
go-app/cmd/vault/README.md
Normal file
|
|
@ -0,0 +1,57 @@
|
||||||
|
# Vault Email Processor
|
||||||
|
|
||||||
|
This is a Go rewrite of the PHP vault.php script that processes emails for the CMC Sales system.
|
||||||
|
|
||||||
|
## Key Changes from PHP Version
|
||||||
|
|
||||||
|
1. **No ripmime dependency**: Uses the enmime Go library for MIME parsing instead of external ripmime binary
|
||||||
|
2. **Better error handling**: Proper error handling and database transactions
|
||||||
|
3. **Type safety**: Strongly typed Go structures
|
||||||
|
4. **Modern email parsing**: Uses enmime for robust email parsing
|
||||||
|
|
||||||
|
## Features
|
||||||
|
|
||||||
|
- Processes emails from vault directory
|
||||||
|
- Parses email headers and extracts recipients
|
||||||
|
- Identifies and creates users as needed
|
||||||
|
- Extracts attachments and email body parts
|
||||||
|
- Matches document identifiers in subjects (enquiries, invoices, POs, jobs)
|
||||||
|
- Saves emails with all associations to database
|
||||||
|
- Moves processed emails to processed directory
|
||||||
|
|
||||||
|
## Usage
|
||||||
|
|
||||||
|
```bash
|
||||||
|
go run cmd/vault/main.go \
|
||||||
|
--emaildir=/var/www/emails \
|
||||||
|
--vaultdir=/var/www/vaultmsgs/new \
|
||||||
|
--processeddir=/var/www/vaultmsgs/cur \
|
||||||
|
--dbhost=127.0.0.1 \
|
||||||
|
--dbuser=cmc \
|
||||||
|
--dbpass="xVRQI&cA?7AU=hqJ!%au" \
|
||||||
|
--dbname=cmc
|
||||||
|
```
|
||||||
|
|
||||||
|
## Build
|
||||||
|
|
||||||
|
```bash
|
||||||
|
go build -o vault cmd/vault/main.go
|
||||||
|
```
|
||||||
|
|
||||||
|
## Dependencies
|
||||||
|
|
||||||
|
- github.com/jhillyerd/enmime - MIME email parsing
|
||||||
|
- github.com/google/uuid - UUID generation for unique filenames
|
||||||
|
- github.com/go-sql-driver/mysql - MySQL driver
|
||||||
|
|
||||||
|
## Database Tables Used
|
||||||
|
|
||||||
|
- emails - Main email records
|
||||||
|
- email_recipients - To/CC recipients
|
||||||
|
- email_attachments - File attachments
|
||||||
|
- emails_enquiries - Email to enquiry associations
|
||||||
|
- emails_invoices - Email to invoice associations
|
||||||
|
- emails_purchase_orders - Email to PO associations
|
||||||
|
- emails_jobs - Email to job associations
|
||||||
|
- users - System users
|
||||||
|
- enquiries, invoices, purchase_orders, jobs - For identifier matching
|
||||||
624
go-app/cmd/vault/main.go
Normal file
624
go-app/cmd/vault/main.go
Normal file
|
|
@ -0,0 +1,624 @@
|
||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"database/sql"
|
||||||
|
"flag"
|
||||||
|
"fmt"
|
||||||
|
"io/ioutil"
|
||||||
|
"log"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"regexp"
|
||||||
|
"strings"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
_ "github.com/go-sql-driver/mysql"
|
||||||
|
"github.com/google/uuid"
|
||||||
|
"github.com/jhillyerd/enmime"
|
||||||
|
)
|
||||||
|
|
||||||
|
type Config struct {
|
||||||
|
EmailDir string
|
||||||
|
VaultDir string
|
||||||
|
ProcessedDir string
|
||||||
|
DBHost string
|
||||||
|
DBUser string
|
||||||
|
DBPassword string
|
||||||
|
DBName string
|
||||||
|
}
|
||||||
|
|
||||||
|
type EmailProcessor struct {
|
||||||
|
db *sql.DB
|
||||||
|
config Config
|
||||||
|
enquiryMap map[string]int
|
||||||
|
invoiceMap map[string]int
|
||||||
|
poMap map[string]int
|
||||||
|
userMap map[string]int
|
||||||
|
jobMap map[string]int
|
||||||
|
}
|
||||||
|
|
||||||
|
type Attachment struct {
|
||||||
|
Type string
|
||||||
|
Name string
|
||||||
|
Filename string
|
||||||
|
Size int64
|
||||||
|
IsMessageBody int
|
||||||
|
}
|
||||||
|
|
||||||
|
func main() {
|
||||||
|
var config Config
|
||||||
|
flag.StringVar(&config.EmailDir, "emaildir", "/var/www/emails", "Email storage directory")
|
||||||
|
flag.StringVar(&config.VaultDir, "vaultdir", "/var/www/vaultmsgs/new", "Vault messages directory")
|
||||||
|
flag.StringVar(&config.ProcessedDir, "processeddir", "/var/www/vaultmsgs/cur", "Processed messages directory")
|
||||||
|
flag.StringVar(&config.DBHost, "dbhost", "127.0.0.1", "Database host")
|
||||||
|
flag.StringVar(&config.DBUser, "dbuser", "cmc", "Database user")
|
||||||
|
flag.StringVar(&config.DBPassword, "dbpass", "xVRQI&cA?7AU=hqJ!%au", "Database password")
|
||||||
|
flag.StringVar(&config.DBName, "dbname", "cmc", "Database name")
|
||||||
|
flag.Parse()
|
||||||
|
|
||||||
|
dsn := fmt.Sprintf("%s:%s@tcp(%s)/%s?parseTime=true",
|
||||||
|
config.DBUser, config.DBPassword, config.DBHost, config.DBName)
|
||||||
|
|
||||||
|
db, err := sql.Open("mysql", dsn)
|
||||||
|
if err != nil {
|
||||||
|
log.Fatal("Failed to connect to database:", err)
|
||||||
|
}
|
||||||
|
defer db.Close()
|
||||||
|
|
||||||
|
processor := &EmailProcessor{
|
||||||
|
db: db,
|
||||||
|
config: config,
|
||||||
|
}
|
||||||
|
|
||||||
|
if err := processor.loadMaps(); err != nil {
|
||||||
|
log.Fatal("Failed to load maps:", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if err := processor.processEmails(); err != nil {
|
||||||
|
log.Fatal("Failed to process emails:", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (p *EmailProcessor) loadMaps() error {
|
||||||
|
p.enquiryMap = make(map[string]int)
|
||||||
|
p.invoiceMap = make(map[string]int)
|
||||||
|
p.poMap = make(map[string]int)
|
||||||
|
p.userMap = make(map[string]int)
|
||||||
|
p.jobMap = make(map[string]int)
|
||||||
|
|
||||||
|
// Load enquiries
|
||||||
|
rows, err := p.db.Query("SELECT id, title FROM enquiries")
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
defer rows.Close()
|
||||||
|
for rows.Next() {
|
||||||
|
var id int
|
||||||
|
var title string
|
||||||
|
if err := rows.Scan(&id, &title); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
p.enquiryMap[title] = id
|
||||||
|
}
|
||||||
|
|
||||||
|
// Load invoices
|
||||||
|
rows, err = p.db.Query("SELECT id, title FROM invoices")
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
defer rows.Close()
|
||||||
|
for rows.Next() {
|
||||||
|
var id int
|
||||||
|
var title string
|
||||||
|
if err := rows.Scan(&id, &title); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
p.invoiceMap[title] = id
|
||||||
|
}
|
||||||
|
|
||||||
|
// Load purchase orders
|
||||||
|
rows, err = p.db.Query("SELECT id, title FROM purchase_orders")
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
defer rows.Close()
|
||||||
|
for rows.Next() {
|
||||||
|
var id int
|
||||||
|
var title string
|
||||||
|
if err := rows.Scan(&id, &title); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
p.poMap[title] = id
|
||||||
|
}
|
||||||
|
|
||||||
|
// Load users
|
||||||
|
rows, err = p.db.Query("SELECT id, email FROM users")
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
defer rows.Close()
|
||||||
|
for rows.Next() {
|
||||||
|
var id int
|
||||||
|
var email string
|
||||||
|
if err := rows.Scan(&id, &email); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
p.userMap[strings.ToLower(email)] = id
|
||||||
|
}
|
||||||
|
|
||||||
|
// Load jobs
|
||||||
|
rows, err = p.db.Query("SELECT id, title FROM jobs")
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
defer rows.Close()
|
||||||
|
for rows.Next() {
|
||||||
|
var id int
|
||||||
|
var title string
|
||||||
|
if err := rows.Scan(&id, &title); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
p.jobMap[title] = id
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (p *EmailProcessor) processEmails() error {
|
||||||
|
files, err := ioutil.ReadDir(p.config.VaultDir)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, file := range files {
|
||||||
|
if file.IsDir() || file.Name() == "." || file.Name() == ".." {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
fmt.Printf("Handling %s\n", file.Name())
|
||||||
|
if err := p.processEmail(file.Name()); err != nil {
|
||||||
|
log.Printf("Error processing %s: %v\n", file.Name(), err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (p *EmailProcessor) processEmail(filename string) error {
|
||||||
|
emailPath := filepath.Join(p.config.VaultDir, filename)
|
||||||
|
content, err := ioutil.ReadFile(emailPath)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
if len(content) == 0 {
|
||||||
|
fmt.Println("No content found. Ignoring this email")
|
||||||
|
return p.moveEmail(filename)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Parse email with enmime
|
||||||
|
env, err := enmime.ReadEnvelope(bytes.NewReader(content))
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
// Get recipients
|
||||||
|
toRecipients := p.parseEnmimeAddresses(env.GetHeader("To"))
|
||||||
|
fromRecipients := p.parseEnmimeAddresses(env.GetHeader("From"))
|
||||||
|
ccRecipients := p.parseEnmimeAddresses(env.GetHeader("Cc"))
|
||||||
|
|
||||||
|
// Check if we should save this email
|
||||||
|
saveThis := false
|
||||||
|
fromKnownUser := false
|
||||||
|
|
||||||
|
for _, email := range toRecipients {
|
||||||
|
if p.userExists(email) {
|
||||||
|
saveThis = true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, email := range fromRecipients {
|
||||||
|
if p.userExists(email) {
|
||||||
|
saveThis = true
|
||||||
|
fromKnownUser = true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, email := range ccRecipients {
|
||||||
|
if p.userExists(email) {
|
||||||
|
saveThis = true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
subject := env.GetHeader("Subject")
|
||||||
|
if subject == "" {
|
||||||
|
fmt.Println("No subject found. Ignoring this email")
|
||||||
|
return p.moveEmail(filename)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Check for identifiers in subject
|
||||||
|
foundEnquiries := p.checkIdentifier(subject, p.enquiryMap, "enquiry")
|
||||||
|
foundInvoices := p.checkIdentifier(subject, p.invoiceMap, "invoice")
|
||||||
|
foundPOs := p.checkIdentifier(subject, p.poMap, "purchaseorder")
|
||||||
|
foundJobs := p.checkIdentifier(subject, p.jobMap, "job")
|
||||||
|
|
||||||
|
foundIdent := len(foundEnquiries) > 0 || len(foundInvoices) > 0 ||
|
||||||
|
len(foundPOs) > 0 || len(foundJobs) > 0
|
||||||
|
|
||||||
|
if fromKnownUser || saveThis || foundIdent {
|
||||||
|
// Process and save the email
|
||||||
|
unixTime := time.Now().Unix()
|
||||||
|
if date, err := env.Date(); err == nil {
|
||||||
|
unixTime = date.Unix()
|
||||||
|
}
|
||||||
|
|
||||||
|
// Get recipient user IDs
|
||||||
|
recipientIDs := make(map[string][]int)
|
||||||
|
recipientIDs["to"] = p.getUserIDs(toRecipients)
|
||||||
|
recipientIDs["from"] = p.getUserIDs(fromRecipients)
|
||||||
|
recipientIDs["cc"] = p.getUserIDs(ccRecipients)
|
||||||
|
|
||||||
|
if len(recipientIDs["from"]) == 0 {
|
||||||
|
fmt.Println("Email has no From Recipient ID. Ignoring this email")
|
||||||
|
return p.moveEmail(filename)
|
||||||
|
}
|
||||||
|
|
||||||
|
fmt.Println("---------START MESSAGE -----------------")
|
||||||
|
fmt.Printf("Subject: %s\n", subject)
|
||||||
|
|
||||||
|
// Extract attachments using enmime
|
||||||
|
relativePath := p.getAttachmentDirectory(unixTime)
|
||||||
|
attachments := p.extractEnmimeAttachments(env, relativePath)
|
||||||
|
|
||||||
|
// Save email to database
|
||||||
|
if err := p.saveEmail(filename, subject, unixTime, recipientIDs, attachments,
|
||||||
|
foundEnquiries, foundInvoices, foundPOs, foundJobs); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
fmt.Println("--------END MESSAGE ------")
|
||||||
|
} else {
|
||||||
|
fmt.Printf("Email will not be saved. Subject: %s\n", subject)
|
||||||
|
}
|
||||||
|
|
||||||
|
return p.moveEmail(filename)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (p *EmailProcessor) parseEnmimeAddresses(header string) []string {
|
||||||
|
var emails []string
|
||||||
|
if header == "" {
|
||||||
|
return emails
|
||||||
|
}
|
||||||
|
|
||||||
|
addresses, err := enmime.ParseAddressList(header)
|
||||||
|
if err != nil {
|
||||||
|
return emails
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, addr := range addresses {
|
||||||
|
emails = append(emails, strings.ToLower(addr.Address))
|
||||||
|
}
|
||||||
|
|
||||||
|
return emails
|
||||||
|
}
|
||||||
|
|
||||||
|
func (p *EmailProcessor) userExists(email string) bool {
|
||||||
|
_, exists := p.userMap[strings.ToLower(email)]
|
||||||
|
return exists
|
||||||
|
}
|
||||||
|
|
||||||
|
func (p *EmailProcessor) getUserIDs(emails []string) []int {
|
||||||
|
var ids []int
|
||||||
|
for _, email := range emails {
|
||||||
|
if id, exists := p.userMap[strings.ToLower(email)]; exists {
|
||||||
|
ids = append(ids, id)
|
||||||
|
} else {
|
||||||
|
// Create new user
|
||||||
|
newID := p.createUser(email)
|
||||||
|
if newID > 0 {
|
||||||
|
ids = append(ids, newID)
|
||||||
|
p.userMap[strings.ToLower(email)] = newID
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return ids
|
||||||
|
}
|
||||||
|
|
||||||
|
func (p *EmailProcessor) createUser(email string) int {
|
||||||
|
fmt.Printf("Making a new User for: '%s'\n", email)
|
||||||
|
|
||||||
|
result, err := p.db.Exec(
|
||||||
|
"INSERT INTO users (type, email, by_vault, created, modified) VALUES (?, ?, ?, NOW(), NOW())",
|
||||||
|
"contact", strings.ToLower(email), 1)
|
||||||
|
|
||||||
|
if err != nil {
|
||||||
|
fmt.Printf("Serious Error: Unable to create user for email '%s': %v\n", email, err)
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
|
||||||
|
id, err := result.LastInsertId()
|
||||||
|
if err != nil {
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
|
||||||
|
fmt.Printf("New User '%s' Added with ID: %d\n", email, id)
|
||||||
|
return int(id)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (p *EmailProcessor) checkIdentifier(subject string, identMap map[string]int, identType string) []int {
|
||||||
|
var results []int
|
||||||
|
var re *regexp.Regexp
|
||||||
|
|
||||||
|
switch identType {
|
||||||
|
case "enquiry":
|
||||||
|
re = regexp.MustCompile(`CMC\d+([NVQWSOT]|ACT|NT)E\d+-\d+`)
|
||||||
|
case "invoice":
|
||||||
|
re = regexp.MustCompile(`CMCIN\d+`)
|
||||||
|
case "purchaseorder":
|
||||||
|
re = regexp.MustCompile(`CMCPO\d+`)
|
||||||
|
case "job":
|
||||||
|
re = regexp.MustCompile(`(JAN|FEB|MAR|APR|MAY|JUN|JUL|AUG|SEP|OCT|NOV|DEC)\d+(N|V|W|S|T|NT|ACT|Q|O)J\d+`)
|
||||||
|
}
|
||||||
|
|
||||||
|
if re != nil {
|
||||||
|
matches := re.FindAllString(subject, -1)
|
||||||
|
for _, match := range matches {
|
||||||
|
if id, exists := identMap[match]; exists {
|
||||||
|
results = append(results, id)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return results
|
||||||
|
}
|
||||||
|
|
||||||
|
func (p *EmailProcessor) getAttachmentDirectory(unixTime int64) string {
|
||||||
|
t := time.Unix(unixTime, 0)
|
||||||
|
monthYear := t.Format("01-2006")
|
||||||
|
path := filepath.Join(p.config.EmailDir, monthYear)
|
||||||
|
|
||||||
|
if err := os.MkdirAll(path, 0755); err != nil {
|
||||||
|
log.Printf("Failed to create directory %s: %v", path, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
return monthYear
|
||||||
|
}
|
||||||
|
|
||||||
|
func (p *EmailProcessor) extractEnmimeAttachments(env *enmime.Envelope, relativePath string) []Attachment {
|
||||||
|
var attachments []Attachment
|
||||||
|
outputDir := filepath.Join(p.config.EmailDir, relativePath)
|
||||||
|
uuid := uuid.New().String()
|
||||||
|
|
||||||
|
// Ensure output directory exists
|
||||||
|
if err := os.MkdirAll(outputDir, 0755); err != nil {
|
||||||
|
log.Printf("Failed to create output directory: %v", err)
|
||||||
|
return attachments
|
||||||
|
}
|
||||||
|
|
||||||
|
biggestHTMLSize := int64(0)
|
||||||
|
biggestHTMLIdx := -1
|
||||||
|
biggestPlainSize := int64(0)
|
||||||
|
biggestPlainIdx := -1
|
||||||
|
|
||||||
|
// Process HTML part if exists
|
||||||
|
if env.HTML != "" {
|
||||||
|
htmlData := []byte(env.HTML)
|
||||||
|
fileName := "texthtml"
|
||||||
|
newFileName := uuid + "-" + fileName
|
||||||
|
filePath := filepath.Join(outputDir, newFileName)
|
||||||
|
|
||||||
|
if err := ioutil.WriteFile(filePath, htmlData, 0644); err == nil {
|
||||||
|
att := Attachment{
|
||||||
|
Type: "text/html",
|
||||||
|
Name: filepath.Join(relativePath, newFileName),
|
||||||
|
Filename: fileName,
|
||||||
|
Size: int64(len(htmlData)),
|
||||||
|
IsMessageBody: 0,
|
||||||
|
}
|
||||||
|
attachments = append(attachments, att)
|
||||||
|
biggestHTMLIdx = len(attachments) - 1
|
||||||
|
biggestHTMLSize = int64(len(htmlData))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Process plain text part if exists
|
||||||
|
if env.Text != "" {
|
||||||
|
textData := []byte(env.Text)
|
||||||
|
fileName := "textplain"
|
||||||
|
newFileName := uuid + "-" + fileName
|
||||||
|
filePath := filepath.Join(outputDir, newFileName)
|
||||||
|
|
||||||
|
if err := ioutil.WriteFile(filePath, textData, 0644); err == nil {
|
||||||
|
att := Attachment{
|
||||||
|
Type: "text/plain",
|
||||||
|
Name: filepath.Join(relativePath, newFileName),
|
||||||
|
Filename: fileName,
|
||||||
|
Size: int64(len(textData)),
|
||||||
|
IsMessageBody: 0,
|
||||||
|
}
|
||||||
|
attachments = append(attachments, att)
|
||||||
|
biggestPlainIdx = len(attachments) - 1
|
||||||
|
biggestPlainSize = int64(len(textData))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Process file attachments
|
||||||
|
for _, part := range env.Attachments {
|
||||||
|
fileName := part.FileName
|
||||||
|
if fileName == "" {
|
||||||
|
fileName = "attachment"
|
||||||
|
}
|
||||||
|
|
||||||
|
newFileName := uuid + "-" + fileName
|
||||||
|
filePath := filepath.Join(outputDir, newFileName)
|
||||||
|
|
||||||
|
if err := ioutil.WriteFile(filePath, part.Content, 0644); err != nil {
|
||||||
|
log.Printf("Failed to save attachment %s: %v", fileName, err)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
att := Attachment{
|
||||||
|
Type: part.ContentType,
|
||||||
|
Name: filepath.Join(relativePath, newFileName),
|
||||||
|
Filename: fileName,
|
||||||
|
Size: int64(len(part.Content)),
|
||||||
|
IsMessageBody: 0,
|
||||||
|
}
|
||||||
|
|
||||||
|
attachments = append(attachments, att)
|
||||||
|
idx := len(attachments) - 1
|
||||||
|
|
||||||
|
// Track largest HTML and plain text attachments
|
||||||
|
if strings.HasPrefix(part.ContentType, "text/html") && int64(len(part.Content)) > biggestHTMLSize {
|
||||||
|
biggestHTMLSize = int64(len(part.Content))
|
||||||
|
biggestHTMLIdx = idx
|
||||||
|
} else if strings.HasPrefix(part.ContentType, "text/plain") && int64(len(part.Content)) > biggestPlainSize {
|
||||||
|
biggestPlainSize = int64(len(part.Content))
|
||||||
|
biggestPlainIdx = idx
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Process inline parts
|
||||||
|
for _, part := range env.Inlines {
|
||||||
|
fileName := part.FileName
|
||||||
|
if fileName == "" {
|
||||||
|
fileName = "inline"
|
||||||
|
}
|
||||||
|
|
||||||
|
newFileName := uuid + "-" + fileName
|
||||||
|
filePath := filepath.Join(outputDir, newFileName)
|
||||||
|
|
||||||
|
if err := ioutil.WriteFile(filePath, part.Content, 0644); err != nil {
|
||||||
|
log.Printf("Failed to save inline part %s: %v", fileName, err)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
att := Attachment{
|
||||||
|
Type: part.ContentType,
|
||||||
|
Name: filepath.Join(relativePath, newFileName),
|
||||||
|
Filename: fileName,
|
||||||
|
Size: int64(len(part.Content)),
|
||||||
|
IsMessageBody: 0,
|
||||||
|
}
|
||||||
|
|
||||||
|
attachments = append(attachments, att)
|
||||||
|
idx := len(attachments) - 1
|
||||||
|
|
||||||
|
// Track largest HTML and plain text attachments
|
||||||
|
if strings.HasPrefix(part.ContentType, "text/html") && int64(len(part.Content)) > biggestHTMLSize {
|
||||||
|
biggestHTMLSize = int64(len(part.Content))
|
||||||
|
biggestHTMLIdx = idx
|
||||||
|
} else if strings.HasPrefix(part.ContentType, "text/plain") && int64(len(part.Content)) > biggestPlainSize {
|
||||||
|
biggestPlainSize = int64(len(part.Content))
|
||||||
|
biggestPlainIdx = idx
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Mark the message body
|
||||||
|
if biggestHTMLIdx >= 0 {
|
||||||
|
attachments[biggestHTMLIdx].IsMessageBody = 1
|
||||||
|
} else if biggestPlainIdx >= 0 {
|
||||||
|
attachments[biggestPlainIdx].IsMessageBody = 1
|
||||||
|
}
|
||||||
|
|
||||||
|
return attachments
|
||||||
|
}
|
||||||
|
|
||||||
|
func (p *EmailProcessor) saveEmail(filename, subject string, unixTime int64,
|
||||||
|
recipientIDs map[string][]int, attachments []Attachment,
|
||||||
|
foundEnquiries, foundInvoices, foundPOs, foundJobs []int) error {
|
||||||
|
|
||||||
|
tx, err := p.db.Begin()
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
defer tx.Rollback()
|
||||||
|
|
||||||
|
// Insert email
|
||||||
|
result, err := tx.Exec(
|
||||||
|
"INSERT INTO emails (user_id, udate, created, subject, filename) VALUES (?, ?, NOW(), ?, ?)",
|
||||||
|
recipientIDs["from"][0], unixTime, subject, filename)
|
||||||
|
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
emailID, err := result.LastInsertId()
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
// Insert recipients
|
||||||
|
for recipType, userIDs := range recipientIDs {
|
||||||
|
for _, userID := range userIDs {
|
||||||
|
if recipType == "from" {
|
||||||
|
continue // From is already stored in emails.user_id
|
||||||
|
}
|
||||||
|
_, err = tx.Exec(
|
||||||
|
"INSERT INTO email_recipients (email_id, user_id, type) VALUES (?, ?, ?)",
|
||||||
|
emailID, userID, recipType)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Insert attachments
|
||||||
|
for _, att := range attachments {
|
||||||
|
_, err = tx.Exec(
|
||||||
|
"INSERT INTO email_attachments (email_id, name, type, size, filename, is_message_body, created) VALUES (?, ?, ?, ?, ?, ?, NOW())",
|
||||||
|
emailID, att.Name, att.Type, att.Size, att.Filename, att.IsMessageBody)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Insert associations
|
||||||
|
for _, jobID := range foundJobs {
|
||||||
|
_, err = tx.Exec("INSERT INTO emails_jobs (email_id, job_id) VALUES (?, ?)", emailID, jobID)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, poID := range foundPOs {
|
||||||
|
_, err = tx.Exec("INSERT INTO emails_purchase_orders (email_id, purchase_order_id) VALUES (?, ?)", emailID, poID)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, enqID := range foundEnquiries {
|
||||||
|
_, err = tx.Exec("INSERT INTO emails_enquiries (email_id, enquiry_id) VALUES (?, ?)", emailID, enqID)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, invID := range foundInvoices {
|
||||||
|
_, err = tx.Exec("INSERT INTO emails_invoices (email_id, invoice_id) VALUES (?, ?)", emailID, invID)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if err := tx.Commit(); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
fmt.Println("Success. We made an email")
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (p *EmailProcessor) moveEmail(filename string) error {
|
||||||
|
oldPath := filepath.Join(p.config.VaultDir, filename)
|
||||||
|
newPath := filepath.Join(p.config.ProcessedDir, filename+":S")
|
||||||
|
|
||||||
|
if err := os.Rename(oldPath, newPath); err != nil {
|
||||||
|
fmt.Printf("Unable to move %s to %s: %v\n", oldPath, newPath, err)
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
@ -4,7 +4,22 @@ go 1.23
|
||||||
|
|
||||||
require (
|
require (
|
||||||
github.com/go-sql-driver/mysql v1.7.1
|
github.com/go-sql-driver/mysql v1.7.1
|
||||||
|
github.com/google/uuid v1.6.0
|
||||||
github.com/gorilla/mux v1.8.1
|
github.com/gorilla/mux v1.8.1
|
||||||
|
github.com/jhillyerd/enmime v1.3.0
|
||||||
github.com/joho/godotenv v1.5.1
|
github.com/joho/godotenv v1.5.1
|
||||||
github.com/jung-kurt/gofpdf v1.16.2
|
github.com/jung-kurt/gofpdf v1.16.2
|
||||||
)
|
)
|
||||||
|
|
||||||
|
require (
|
||||||
|
github.com/cention-sany/utf7 v0.0.0-20170124080048-26cad61bd60a // indirect
|
||||||
|
github.com/gogs/chardet v0.0.0-20211120154057-b7413eaefb8f // indirect
|
||||||
|
github.com/jaytaylor/html2text v0.0.0-20230321000545-74c2419ad056 // indirect
|
||||||
|
github.com/mattn/go-runewidth v0.0.15 // indirect
|
||||||
|
github.com/olekukonko/tablewriter v0.0.5 // indirect
|
||||||
|
github.com/pkg/errors v0.9.1 // indirect
|
||||||
|
github.com/rivo/uniseg v0.4.4 // indirect
|
||||||
|
github.com/ssor/bom v0.0.0-20170718123548-6386211fdfcf // indirect
|
||||||
|
golang.org/x/net v0.23.0 // indirect
|
||||||
|
golang.org/x/text v0.14.0 // indirect
|
||||||
|
)
|
||||||
|
|
|
||||||
|
|
@ -1,18 +1,44 @@
|
||||||
github.com/boombuler/barcode v1.0.0/go.mod h1:paBWMcWSl3LHKBqUq+rly7CNSldXjb2rDl3JlRe0mD8=
|
github.com/boombuler/barcode v1.0.0/go.mod h1:paBWMcWSl3LHKBqUq+rly7CNSldXjb2rDl3JlRe0mD8=
|
||||||
|
github.com/cention-sany/utf7 v0.0.0-20170124080048-26cad61bd60a h1:MISbI8sU/PSK/ztvmWKFcI7UGb5/HQT7B+i3a2myKgI=
|
||||||
|
github.com/cention-sany/utf7 v0.0.0-20170124080048-26cad61bd60a/go.mod h1:2GxOXOlEPAMFPfp014mK1SWq8G8BN8o7/dfYqJrVGn8=
|
||||||
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||||
github.com/go-sql-driver/mysql v1.7.1 h1:lUIinVbN1DY0xBg0eMOzmmtGoHwWBbvnWubQUrtU8EI=
|
github.com/go-sql-driver/mysql v1.7.1 h1:lUIinVbN1DY0xBg0eMOzmmtGoHwWBbvnWubQUrtU8EI=
|
||||||
github.com/go-sql-driver/mysql v1.7.1/go.mod h1:OXbVy3sEdcQ2Doequ6Z5BW6fXNQTmx+9S1MCJN5yJMI=
|
github.com/go-sql-driver/mysql v1.7.1/go.mod h1:OXbVy3sEdcQ2Doequ6Z5BW6fXNQTmx+9S1MCJN5yJMI=
|
||||||
|
github.com/gogs/chardet v0.0.0-20211120154057-b7413eaefb8f h1:3BSP1Tbs2djlpprl7wCLuiqMaUh5SJkkzI2gDs+FgLs=
|
||||||
|
github.com/gogs/chardet v0.0.0-20211120154057-b7413eaefb8f/go.mod h1:Pcatq5tYkCW2Q6yrR2VRHlbHpZ/R4/7qyL1TCF7vl14=
|
||||||
|
github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
|
||||||
|
github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
|
||||||
github.com/gorilla/mux v1.8.1 h1:TuBL49tXwgrFYWhqrNgrUNEY92u81SPhu7sTdzQEiWY=
|
github.com/gorilla/mux v1.8.1 h1:TuBL49tXwgrFYWhqrNgrUNEY92u81SPhu7sTdzQEiWY=
|
||||||
github.com/gorilla/mux v1.8.1/go.mod h1:AKf9I4AEqPTmMytcMc0KkNouC66V3BtZ4qD5fmWSiMQ=
|
github.com/gorilla/mux v1.8.1/go.mod h1:AKf9I4AEqPTmMytcMc0KkNouC66V3BtZ4qD5fmWSiMQ=
|
||||||
|
github.com/jaytaylor/html2text v0.0.0-20230321000545-74c2419ad056 h1:iCHtR9CQyktQ5+f3dMVZfwD2KWJUgm7M0gdL9NGr8KA=
|
||||||
|
github.com/jaytaylor/html2text v0.0.0-20230321000545-74c2419ad056/go.mod h1:CVKlgaMiht+LXvHG173ujK6JUhZXKb2u/BQtjPDIvyk=
|
||||||
|
github.com/jhillyerd/enmime v1.3.0 h1:LV5kzfLidiOr8qRGIpYYmUZCnhrPbcFAnAFUnWn99rw=
|
||||||
|
github.com/jhillyerd/enmime v1.3.0/go.mod h1:6c6jg5HdRRV2FtvVL69LjiX1M8oE0xDX9VEhV3oy4gs=
|
||||||
github.com/joho/godotenv v1.5.1 h1:7eLL/+HRGLY0ldzfGMeQkb7vMd0as4CfYvUVzLqw0N0=
|
github.com/joho/godotenv v1.5.1 h1:7eLL/+HRGLY0ldzfGMeQkb7vMd0as4CfYvUVzLqw0N0=
|
||||||
github.com/joho/godotenv v1.5.1/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4=
|
github.com/joho/godotenv v1.5.1/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4=
|
||||||
github.com/jung-kurt/gofpdf v1.0.0/go.mod h1:7Id9E/uU8ce6rXgefFLlgrJj/GYY22cpxn+r32jIOes=
|
github.com/jung-kurt/gofpdf v1.0.0/go.mod h1:7Id9E/uU8ce6rXgefFLlgrJj/GYY22cpxn+r32jIOes=
|
||||||
github.com/jung-kurt/gofpdf v1.16.2 h1:jgbatWHfRlPYiK85qgevsZTHviWXKwB1TTiKdz5PtRc=
|
github.com/jung-kurt/gofpdf v1.16.2 h1:jgbatWHfRlPYiK85qgevsZTHviWXKwB1TTiKdz5PtRc=
|
||||||
github.com/jung-kurt/gofpdf v1.16.2/go.mod h1:1hl7y57EsiPAkLbOwzpzqgx1A30nQCk/YmFV8S2vmK0=
|
github.com/jung-kurt/gofpdf v1.16.2/go.mod h1:1hl7y57EsiPAkLbOwzpzqgx1A30nQCk/YmFV8S2vmK0=
|
||||||
|
github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI=
|
||||||
|
github.com/mattn/go-runewidth v0.0.15 h1:UNAjwbU9l54TA3KzvqLGxwWjHmMgBUVhBiTjelZgg3U=
|
||||||
|
github.com/mattn/go-runewidth v0.0.15/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w=
|
||||||
|
github.com/olekukonko/tablewriter v0.0.5 h1:P2Ga83D34wi1o9J6Wh1mRuqd4mF/x/lgBS7N7AbDhec=
|
||||||
|
github.com/olekukonko/tablewriter v0.0.5/go.mod h1:hPp6KlRPjbx+hW8ykQs1w3UBbZlj6HuIJcUGPhkA7kY=
|
||||||
github.com/phpdave11/gofpdi v1.0.7/go.mod h1:vBmVV0Do6hSBHC8uKUQ71JGW+ZGQq74llk/7bXwjDoI=
|
github.com/phpdave11/gofpdi v1.0.7/go.mod h1:vBmVV0Do6hSBHC8uKUQ71JGW+ZGQq74llk/7bXwjDoI=
|
||||||
github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
|
github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
|
||||||
|
github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
|
||||||
|
github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
|
||||||
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
||||||
|
github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
|
||||||
|
github.com/rivo/uniseg v0.4.4 h1:8TfxU8dW6PdqD27gjM8MVNuicgxIjxpm4K7x4jp8sis=
|
||||||
|
github.com/rivo/uniseg v0.4.4/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88=
|
||||||
github.com/ruudk/golang-pdf417 v0.0.0-20181029194003-1af4ab5afa58/go.mod h1:6lfFZQK844Gfx8o5WFuvpxWRwnSoipWe/p622j1v06w=
|
github.com/ruudk/golang-pdf417 v0.0.0-20181029194003-1af4ab5afa58/go.mod h1:6lfFZQK844Gfx8o5WFuvpxWRwnSoipWe/p622j1v06w=
|
||||||
|
github.com/ssor/bom v0.0.0-20170718123548-6386211fdfcf h1:pvbZ0lM0XWPBqUKqFU8cmavspvIl9nulOYwdy6IFRRo=
|
||||||
|
github.com/ssor/bom v0.0.0-20170718123548-6386211fdfcf/go.mod h1:RJID2RhlZKId02nZ62WenDCkgHFerpIOmW0iT7GKmXM=
|
||||||
github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
|
github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
|
||||||
golang.org/x/image v0.0.0-20190910094157-69e4b8554b2a/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
|
golang.org/x/image v0.0.0-20190910094157-69e4b8554b2a/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
|
||||||
|
golang.org/x/net v0.23.0 h1:7EYJ93RZ9vYSZAIb2x3lnuvqO5zneoD6IvWjuhfxjTs=
|
||||||
|
golang.org/x/net v0.23.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg=
|
||||||
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
||||||
|
golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ=
|
||||||
|
golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue