-- ============================================================================
-- Fix Corrupted Character Data in CMC Database
-- ============================================================================
-- This script fixes mojibake (corrupted UTF-8 characters) in the database
-- Run after converting tables to utf8mb4
--
-- IMPORTANT: Review this script and test on a backup first!
-- Create backup: docker exec cmc-db mariadb-dump -u root -psecureRootPassword cmc | gzip > backup_before_fix.sql.gz
--
-- To run: docker exec -i cmc-db mariadb -u root -psecureRootPassword --default-character-set=utf8mb4 cmc < scripts/fix_corrupted_data.sql
-- ============================================================================

-- Create a backup table for principles (optional safety measure)
-- DROP TABLE IF EXISTS principles_backup;
-- CREATE TABLE principles_backup AS SELECT * FROM principles;

SET NAMES utf8mb4;

-- ============================================================================
-- PART 1: Global Pattern Fixes Using REPLACE
-- ============================================================================
-- These fix common mojibake patterns across all affected tables

-- ----------------------------------------------------------------------------
-- Fix CUSTOMERS table
-- ----------------------------------------------------------------------------
UPDATE customers SET name = REPLACE(name, 'â€™', ''') WHERE name LIKE '%â€™%';  -- Smart apostrophe
UPDATE customers SET name = REPLACE(name, 'â€œ', '"') WHERE name LIKE '%â€œ%';  -- Left double quote
UPDATE customers SET name = REPLACE(name, 'â€', '"') WHERE name LIKE '%â€%';  -- Right double quote
UPDATE customers SET name = REPLACE(name, 'â€"', '–') WHERE name LIKE '%â€"%';  -- En dash
UPDATE customers SET name = REPLACE(name, 'â€"', '—') WHERE name LIKE '%â€"%';  -- Em dash
UPDATE customers SET name = REPLACE(name, 'â€‹', '') WHERE name LIKE '%â€‹%';   -- Zero-width space (remove)
UPDATE customers SET name = REPLACE(name, 'â€ƒ', ' ') WHERE name LIKE '%â€ƒ%';  -- Special space
UPDATE customers SET name = REPLACE(name, 'Ã©', 'é') WHERE name LIKE '%Ã©%';   -- é
UPDATE customers SET name = REPLACE(name, 'Ã‰', 'É') WHERE name LIKE '%Ã‰%';   -- É
UPDATE customers SET name = REPLACE(name, 'Ã³', 'ó') WHERE name LIKE '%Ã³%';   -- ó
UPDATE customers SET name = REPLACE(name, 'Ã', 'í') WHERE name LIKE '%Ã%';   -- í
UPDATE customers SET name = REPLACE(name, 'Ã§', 'ç') WHERE name LIKE '%Ã§%';   -- ç
UPDATE customers SET name = REPLACE(name, 'Ã¼', 'ü') WHERE name LIKE '%Ã¼%';   -- ü
UPDATE customers SET name = REPLACE(name, 'Ã¡', 'á') WHERE name LIKE '%Ã¡%';   -- á
UPDATE customers SET name = REPLACE(name, 'Ã±', 'ñ') WHERE name LIKE '%Ã±%';   -- ñ
UPDATE customers SET name = REPLACE(name, 'Ã¶', 'ö') WHERE name LIKE '%Ã¶%';   -- ö
UPDATE customers SET name = REPLACE(name, 'Ã´', 'ô') WHERE name LIKE '%Ã´%';   -- ô
UPDATE customers SET name = REPLACE(name, 'Ä°', 'İ') WHERE name LIKE '%Ä°%';   -- Turkish İ
UPDATE customers SET name = REPLACE(name, 'Å', 'Ş') WHERE name LIKE '%Å%';   -- Turkish Ş
UPDATE customers SET name = REPLACE(name, 'Ä', 'Ğ') WHERE name LIKE '%Ä%';   -- Turkish Ğ
UPDATE customers SET name = REPLACE(name, 'Ä', 'Č') WHERE name LIKE '%Ä%';   -- Czech Č
UPDATE customers SET name = REPLACE(name, 'Ä›', 'ě') WHERE name LIKE '%Ä›%';   -- Czech ě

-- Remove trailing tabs
UPDATE customers SET name = TRIM(TRAILING '\t' FROM name) WHERE name LIKE '%\t';
UPDATE customers SET trading_name = TRIM(TRAILING '\t' FROM trading_name) WHERE trading_name LIKE '%\t';

-- ----------------------------------------------------------------------------
-- Fix PRINCIPLES table
-- ----------------------------------------------------------------------------
UPDATE principles SET name = REPLACE(name, 'Ã¼', 'ü') WHERE name LIKE '%Ã¼%';
UPDATE principles SET name = REPLACE(name, 'Ã©', 'é') WHERE name LIKE '%Ã©%';
UPDATE principles SET name = REPLACE(name, 'Ã³', 'ó') WHERE name LIKE '%Ã³%';

UPDATE principles SET address = REPLACE(address, 'ÃŸ', 'ß') WHERE address LIKE '%ÃŸ%';  -- German ß
UPDATE principles SET address = REPLACE(address, 'Ã¼', 'ü') WHERE address LIKE '%Ã¼%';
UPDATE principles SET address = REPLACE(address, 'Ã©', 'é') WHERE address LIKE '%Ã©%';
UPDATE principles SET address = REPLACE(address, 'Ã±', 'ñ') WHERE address LIKE '%Ã±%';
UPDATE principles SET address = REPLACE(address, 'Ä', 'č') WHERE address LIKE '%Ä%';   -- Czech č
UPDATE principles SET address = REPLACE(address, 'Å¾', 'ž') WHERE address LIKE '%Å¾%';  -- Czech ž
UPDATE principles SET address = REPLACE(address, 'Å¡', 'š') WHERE address LIKE '%Å¡%';  -- Czech š
UPDATE principles SET address = REPLACE(address, 'Ä›', 'ě') WHERE address LIKE '%Ä›%';  -- Czech ě
UPDATE principles SET address = REPLACE(address, 'Å', 'ň') WHERE address LIKE '%Å%';  -- Czech ň
UPDATE principles SET address = REPLACE(address, 'Å™', 'ř') WHERE address LIKE '%Å™%';  -- Czech ř

UPDATE principles SET city = REPLACE(city, 'Ã¼', 'ü') WHERE city LIKE '%Ã¼%';
UPDATE principles SET city = REPLACE(city, 'Ã©', 'é') WHERE city LIKE '%Ã©%';
UPDATE principles SET city = REPLACE(city, 'Ã³', 'ó') WHERE city LIKE '%Ã³%';

-- ----------------------------------------------------------------------------
-- Fix ADDRESSES table
-- ----------------------------------------------------------------------------
UPDATE addresses SET address = REPLACE(address, 'â€"', '–') WHERE address LIKE '%â€"%';  -- En dash
UPDATE addresses SET address = REPLACE(address, 'â€"', '—') WHERE address LIKE '%â€"%';  -- Em dash
UPDATE addresses SET address = REPLACE(address, 'Ã©', 'é') WHERE address LIKE '%Ã©%';
UPDATE addresses SET address = REPLACE(address, 'Ã¼', 'ü') WHERE address LIKE '%Ã¼%';

UPDATE addresses SET city = REPLACE(city, 'Ã©', 'é') WHERE city LIKE '%Ã©%';
UPDATE addresses SET city = REPLACE(city, 'Ã¼', 'ü') WHERE city LIKE '%Ã¼%';

-- ============================================================================
-- PART 2: Specific Record Fixes
-- ============================================================================
-- These fix specific known corrupted records where pattern matching isn't enough

-- ----------------------------------------------------------------------------
-- PRINCIPLES - Specific fixes
-- ----------------------------------------------------------------------------

-- ID 2: German ß
UPDATE principles
SET address = 'Heinz-Fangman-Straße 18'
WHERE id = 2;

-- ID 9: Czech characters (this is complex, needs full correction)
UPDATE principles
SET address = 'Bezručova 2901\n756 61 Rožnov pod Radhoštěm',
    city = 'Rožnov pod Radhoštěm'
WHERE id = 9;

-- ID 13: German ü
UPDATE principles
SET name = 'IEP Technologies GmbH - BRILEX Gesellschaft für Explosionsschutz mbH'
WHERE id = 13;

-- ID 14: German ß
UPDATE principles
SET address = 'Liebigstraße 2'
WHERE id = 14;

-- ID 58: Spanish ó
UPDATE principles
SET name = 'Nosia S.r.l.- Señalización Industrial'
WHERE id = 58;

-- ID 65: German ß
UPDATE principles
SET address = 'Alte Emser Straße 32'
WHERE id = 65;

-- ----------------------------------------------------------------------------
-- CUSTOMERS - Specific fixes
-- ----------------------------------------------------------------------------

-- ID 253: É
UPDATE customers
SET name = 'SOLUZÉ CIVIL ENGINEERS Trading Under SOLUZE PTY LTD'
WHERE id = 253;

-- ID 1006: Smart apostrophe
UPDATE customers
SET name = 'Dr. Prem''s Molecules Private Limited (DPMolecules)'
WHERE id = 1006;

-- ID 1387: Smart quotes
UPDATE customers
SET name = 'DEE ENTERPRISES (QLD) PTY LTD trading as "MEKLEK"'
WHERE id = 1387;

-- ID 1608: Smart apostrophe
UPDATE customers
SET name = 'Guidera O''Connor Pty Ltd'
WHERE id = 1608;

-- ID 2174: Zero-width space
UPDATE customers
SET name = 'Ingredion ANZ Pty Ltd'
WHERE id = 2174;

-- ID 2215: French é
UPDATE customers
SET name = 'Vale Nouvelle-Calédonie S.A.S'
WHERE id = 2215;

-- ID 2375: Spanish ó
UPDATE customers
SET name = 'Evaluación y Control Ambiental S.A.S.'
WHERE id = 2375;

-- ID 3143: Portuguese ç and ó
UPDATE customers
SET name = 'Zontahevy Comércio e Serviços Offshore Ltda'
WHERE id = 3143;

-- ID 3529: French é
UPDATE customers
SET name = 'Société des Mines de Syama'
WHERE id = 3529;

-- ID 3633: Special space (em space)
UPDATE customers
SET name = 'P.J. Berriman & Co PTY LTD'
WHERE id = 3633;

-- ID 4325: Danish ø
UPDATE customers
SET name = 'Rambøll Danmark'
WHERE id = 4325;

-- ID 4350: Turkish characters İ, Ş, Ğ
UPDATE customers
SET name = 'SONNIVA ENERGY MAKİNA İNŞAAT İTH. İHR. LTD. ŞTİ.'
WHERE id = 4350;

-- ID 4669: French é
UPDATE customers
SET name = 'F.C.C. Fluides Conseils Calédonie SARL'
WHERE id = 4669;

-- ID 4743: Czech ě
UPDATE customers
SET name = 'DGPack Prostějov'
WHERE id = 4743;

-- ID 4764: Zero-width space
UPDATE customers
SET name = 'Mccready Welding Pty Ltd Trading under the entity Mccready''s Welding Services'
WHERE id = 4764;

-- ID 4893: Spanish í
UPDATE customers
SET name = 'Oxiquímica, S.A.P.I. de C.V.'
WHERE id = 4893;

-- ----------------------------------------------------------------------------
-- ADDRESSES - Specific fixes
-- ----------------------------------------------------------------------------

-- ID 19: En dash (7–11)
UPDATE addresses
SET address = 'Lvl 3, Building B, 7–11 Talavera Road'
WHERE id = 19;

-- ============================================================================
-- PART 3: Fix PRODUCTS Table (12,536+ corrupted records)
-- ============================================================================
-- Products table has extensive corruption from mojibake and HTML entities

-- Degree symbol: Â° → °
UPDATE products SET title = REPLACE(title, 'Â°', '°') WHERE title LIKE '%Â°%';
UPDATE products SET description = REPLACE(description, 'Â°', '°') WHERE description LIKE '%Â°%';
UPDATE products SET item_description = REPLACE(item_description, 'Â°', '°') WHERE item_description LIKE '%Â°%';

-- Registered trademark: Â® → ®
UPDATE products SET title = REPLACE(title, 'Â®', '®') WHERE title LIKE '%Â®%';
UPDATE products SET description = REPLACE(description, 'Â®', '®') WHERE description LIKE '%Â®%';
UPDATE products SET item_description = REPLACE(item_description, 'Â®', '®') WHERE item_description LIKE '%Â®%';

-- Trademark: â„¢ → ™
UPDATE products SET title = REPLACE(title, 'â„¢', '™') WHERE title LIKE '%â„¢%';
UPDATE products SET description = REPLACE(description, 'â„¢', '™') WHERE description LIKE '%â„¢%';
UPDATE products SET item_description = REPLACE(item_description, 'â„¢', '™') WHERE item_description LIKE '%â„¢%';

-- Smart quotes and dashes in products
UPDATE products SET title = REPLACE(title, 'â€™', ''') WHERE title LIKE '%â€™%';
UPDATE products SET description = REPLACE(description, 'â€™', ''') WHERE description LIKE '%â€™%';
UPDATE products SET title = REPLACE(title, 'â€œ', '"') WHERE title LIKE '%â€œ%';
UPDATE products SET description = REPLACE(description, 'â€œ', '"') WHERE description LIKE '%â€œ%';
UPDATE products SET title = REPLACE(title, 'â€', '"') WHERE title LIKE '%â€%';
UPDATE products SET description = REPLACE(description, 'â€', '"') WHERE description LIKE '%â€%';
UPDATE products SET title = REPLACE(title, 'â€"', '–') WHERE title LIKE '%â€"%';
UPDATE products SET description = REPLACE(description, 'â€"', '–') WHERE description LIKE '%â€"%';
UPDATE products SET title = REPLACE(title, 'â€"', '—') WHERE title LIKE '%â€"%';
UPDATE products SET description = REPLACE(description, 'â€"', '—') WHERE description LIKE '%â€"%';

-- Accented characters in products
UPDATE products SET title = REPLACE(title, 'Ã©', 'é') WHERE title LIKE '%Ã©%';
UPDATE products SET description = REPLACE(description, 'Ã©', 'é') WHERE description LIKE '%Ã©%';
UPDATE products SET title = REPLACE(title, 'Ã¼', 'ü') WHERE title LIKE '%Ã¼%';
UPDATE products SET description = REPLACE(description, 'Ã¼', 'ü') WHERE description LIKE '%Ã¼%';

-- HTML entities (shouldn't be in database)
UPDATE products SET title = REPLACE(title, '&ndash;', '–') WHERE title LIKE '%&ndash;%';
UPDATE products SET description = REPLACE(description, '&ndash;', '–') WHERE description LIKE '%&ndash;%';
UPDATE products SET title = REPLACE(title, '&rdquo;', '"') WHERE title LIKE '%&rdquo;%';
UPDATE products SET description = REPLACE(description, '&rdquo;', '"') WHERE description LIKE '%&rdquo;%';
UPDATE products SET title = REPLACE(title, '&ldquo;', '"') WHERE title LIKE '%&ldquo;%';
UPDATE products SET description = REPLACE(description, '&ldquo;', '"') WHERE description LIKE '%&ldquo;%';
UPDATE products SET title = REPLACE(title, '&amp;', '&') WHERE title LIKE '%&amp;%';
UPDATE products SET description = REPLACE(description, '&amp;', '&') WHERE description LIKE '%&amp;%';
UPDATE products SET title = REPLACE(title, '&frac12;', '½') WHERE title LIKE '%&frac12;%';
UPDATE products SET description = REPLACE(description, '&frac12;', '½') WHERE description LIKE '%&frac12;%';
UPDATE products SET title = REPLACE(title, '&deg;', '°') WHERE title LIKE '%&deg;%';
UPDATE products SET description = REPLACE(description, '&deg;', '°') WHERE description LIKE '%&deg;%';
UPDATE products SET title = REPLACE(title, '&nbsp;', ' ') WHERE title LIKE '%&nbsp;%';
UPDATE products SET description = REPLACE(description, '&nbsp;', ' ') WHERE description LIKE '%&nbsp;%';

-- Specific product fixes
UPDATE products SET title = 'C95SN189C DSTGL40/C-Digital temperature probe for P8xx1 Web Sensor. Range -30 to +80°C. With CINCH connector, 1m Cable' WHERE id = 30;
UPDATE products SET title = 'Mid-West Instrument Model 240 – SC – 02 – O(TT)' WHERE id = 38;
UPDATE products SET title = 'Newson Gale Earth-Rite® II FIBC Static Earthing System, GRP Enclosure with 5m 2 Core Spiral Cable and FIBC Clamp' WHERE id = 67;
UPDATE products SET title = 'Newson Gale Earth-Rite® RTR™ Tri-Mode Static Grounding System, Metal Enclosure, X90-IP Heavy Duty Clamp with 10m 2 Core Spiral Cable' WHERE id = 85;

-- ============================================================================
-- PART 4: Verification Queries
-- ============================================================================
-- Run these after the fixes to verify they worked

-- Check fixed principles
SELECT id, name, city FROM principles WHERE id IN (2, 9, 13, 14, 58, 65);

-- Check fixed customers
SELECT id, name FROM customers WHERE id IN (253, 1006, 1387, 1608, 2174, 2215, 2375, 3143, 3529, 4325, 4350, 4669, 4743, 4764, 4893);

-- Check fixed addresses
SELECT id, address FROM addresses WHERE id = 19;

-- Count remaining corrupted records (should be much lower)
SELECT COUNT(*) as remaining_corrupted_principles
FROM principles
WHERE name REGEXP '[^ -~]' OR address REGEXP '[^ -~]' OR city REGEXP '[^ -~]';

SELECT COUNT(*) as remaining_corrupted_customers
FROM customers
WHERE name REGEXP '[^ -~]';

SELECT COUNT(*) as remaining_corrupted_addresses
FROM addresses
WHERE address REGEXP '[^ -~]' OR city REGEXP '[^ -~]';

SELECT COUNT(*) as remaining_corrupted_products
FROM products
WHERE title REGEXP '[^ -~]' OR description REGEXP '[^ -~]' OR item_description REGEXP '[^ -~]';

-- Check sample fixed products
SELECT id, title FROM products WHERE id IN (30, 38, 67, 85);

-- ============================================================================
-- COMPLETION
-- ============================================================================
SELECT 'Fix script completed! Review the verification queries above.' as status;
