Add support for UTF8 to XDMoD

ubccr · Nov 26, 2024 · e805cf9 · e805cf9
1 parent 77a0b94
commit e805cf9
Show file tree

Hide file tree

Showing 223 changed files with 246 additions and 233 deletions.
diff --git a/classes/ETL/DbModel/Table.php b/classes/ETL/DbModel/Table.php
@@ -48,10 +48,10 @@ class Table extends SchemaEntity implements iEntity, iDiscoverableEntity, iAlter
         'engine'   => null,
 
         // Optional table default character set
-        'charset'  => null,
+        'charset'  => 'utf8',
 
         // Optional table collation
-        'collation' => null,
+        'collation' => 'utf8_unicode_ci',
 
         // Associative array where the keys are column names and the values are Column objects
         'columns'  => array(),

diff --git a/classes/OpenXdmod/Shredder/Slurm.php b/classes/OpenXdmod/Shredder/Slurm.php
@@ -295,9 +295,6 @@ public function shredLine($line)
             return;
         }
 
-        // Convert job name encoding.
-        $job['job_name'] = mb_convert_encoding($job['job_name'], 'ISO-8859-1', 'UTF-8');
-
         // Convert datetime strings into unix timestamps.
         $dateKeys = array(
             'submit_time',

diff --git a/configuration/etl/etl_tables.d/logger/log_level.json b/configuration/etl/etl_tables.d/logger/log_level.json
@@ -3,8 +3,8 @@
         "schema": "mod_logger",
         "name": "log_level",
         "engine": "InnoDB",
-        "charset": "latin1",
-        "collation": "latin1_swedish_ci",
+        "charset": "utf8",
+        "collation": "utf8_unicode_ci",
         "columns": [
             {
                 "name": "log_level_id",

diff --git a/configuration/etl/etl_tables.d/logger/log_table.json b/configuration/etl/etl_tables.d/logger/log_table.json
@@ -3,8 +3,8 @@
         "schema": "mod_logger",
         "name": "log_table",
         "engine": "InnoDB",
-        "charset": "latin1",
-        "collation": "latin1_swedish_ci",
+        "charset": "utf8",
+        "collation": "utf8_unicode_ci",
         "columns": [
             {
                 "name": "id",

diff --git a/tests/artifacts/xdmod/controllers/output/sab_user_enum_tg_users.json b/tests/artifacts/xdmod/controllers/output/sab_user_enum_tg_users.json
@@ -22,7 +22,7 @@
         {
             "id": 4,
             "person_id": "1",
-            "person_name": "Accentor, Alpine"
+            "person_name": "Accentor, Alpiné"
         },
         {
             "id": 5,

diff --git a/tests/artifacts/xdmod/etlv2/configuration/input/etl_tables_8.0.0.d/db_model_test_2.json b/tests/artifacts/xdmod/etlv2/configuration/input/etl_tables_8.0.0.d/db_model_test_2.json
@@ -3,16 +3,22 @@
   "table_definition": {
     "name": "modify_table_test",
     "engine": "MyISAM",
+    "charset": "latin1",
+    "collation": "latin1_swedish_ci",
     "columns": [
       {
         "#": "New column at start of list",
         "name": "new_column_1",
         "type": "varchar(40)",
+        "charset": "latin1",
+        "collation": "latin1_swedish_ci",
         "nullable": true
       },
       {
         "name": "resource",
         "type": "varchar(40)",
+        "charset": "latin1",
+        "collation": "latin1_swedish_ci",
         "nullable": true
       },
       {

diff --git a/tests/artifacts/xdmod/etlv2/configuration/input/etl_tables_8.0.0.d/db_model_test_3.json b/tests/artifacts/xdmod/etlv2/configuration/input/etl_tables_8.0.0.d/db_model_test_3.json
@@ -3,16 +3,22 @@
   "table_definition": {
     "name": "modify_table_test",
     "engine": "MyISAM",
+    "charset": "latin1",
+    "collation": "latin1_swedish_ci",
     "columns": [
       {
         "name": "resource",
         "type": "varchar(40)",
+        "charset": "latin1",
+        "collation": "latin1_swedish_ci",
         "nullable": true
       },
       {
         "#": "This was the 1st column",
         "name": "new_column_1",
         "type": "varchar(40)",
+        "charset": "latin1",
+        "collation": "latin1_swedish_ci",
         "nullable": true
       },
       {

diff --git a/tests/artifacts/xdmod/etlv2/configuration/input/etl_tables_8.0.0.d/db_model_test_4.json b/tests/artifacts/xdmod/etlv2/configuration/input/etl_tables_8.0.0.d/db_model_test_4.json
@@ -3,15 +3,21 @@
   "table_definition": {
     "name": "modify_table_test",
     "engine": "MyISAM",
+    "charset": "latin1",
+    "collation": "latin1_swedish_ci",
     "columns": [
       {
         "name": "resource",
         "type": "varchar(40)",
+        "charset": "latin1",
+        "collation": "latin1_swedish_ci",
         "nullable": true
       },
       {
         "name": "new_column_1",
         "type": "varchar(40)",
+        "charset": "latin1",
+        "collation": "latin1_swedish_ci",
         "nullable": true
       },
       {

diff --git a/tests/artifacts/xdmod/etlv2/configuration/output/create_table.json b/tests/artifacts/xdmod/etlv2/configuration/output/create_table.json
@@ -1,13 +1,13 @@
 {
   "comment": "",
   "engine": "myisam",
-  "charset": "latin1",
-  "collation": "latin1_swedish_ci",
+  "charset": "utf8",
+  "collation": "utf8_unicode_ci",
   "columns": [
     {
       "type": "varchar(40)",
-      "charset": "latin1",
-      "collation": "latin1_swedish_ci",
+      "charset": "utf8",
+      "collation": "utf8_unicode_ci",
       "nullable": true,
       "default": null,
       "extra": null,

diff --git a/tests/artifacts/xdmod/etlv2/dbmodel/output/resourceallocationfact_by.aggregation.sql b/tests/artifacts/xdmod/etlv2/dbmodel/output/resourceallocationfact_by.aggregation.sql
@@ -15,4 +15,4 @@ CREATE TABLE IF NOT EXISTS `resourceallocationfact_by_quarter` (
   INDEX `index_year` (`year`),
   INDEX `index_organization_id` (`organization_id`),
   INDEX `index_resource_id` (`resource_id`)
-) ENGINE = myisam COMMENT = 'Resource Allocation facts aggregated by quarter.';
+) ENGINE = myisam CHARSET = utf8 COLLATE = utf8_unicode_ci COMMENT = 'Resource Allocation facts aggregated by quarter.';
diff --git a/tests/artifacts/xdmod/referencedata/names.csv b/tests/artifacts/xdmod/referencedata/names.csv
@@ -517,7 +517,7 @@ refbl,Red-flanked,Bluetail
 caspl,Caspian,Plover
 egrhula,Snowy,Egret
 whtsp,White-throated,Sparrow
-alpac,Alpine,Accentor
+alpac,Alpiné,Accentor
 allga,Allen's,Gallinule
 glwgu,Glaucous-winged,Gull
 rolle,,Roller

diff --git a/tests/artifacts/xdmod/regression/chartFilterTests.json b/tests/artifacts/xdmod/regression/chartFilterTests.json
@@ -102,12 +102,12 @@
             "date": "2016-12-31",
             "filter_dimension": "pi",
             "filter_values": [
-                "Accentor, Alpine",
+                "Accentor, Alpiné",
                 "Bittern"
             ]
         },
         "expected": {
-            "subtitle": "PI = ( Accentor, Alpine,  Bittern )",
+            "subtitle": "PI = ( Accentor, Alpiné,  Bittern )",
             "yvalue": 3499.8997
         }
     },

diff --git a/...ession/current/expected/reference/Jobs/pi/active_person_count/aggregate-Day-reference.csv b/...ession/current/expected/reference/Jobs/pi/active_person_count/aggregate-Day-reference.csv
@@ -19,7 +19,7 @@ Fulmar,2
 "Shearwater, Balearic",2
 "Warbler, Cetti's",2
 "Warbler, Hooded",2
-"Accentor, Alpine",1
+"Accentor, Alpiné",1
 Bittern,1
 Blackbird,1
 "Bluetail, Red-flanked",1

diff --git a/...sion/current/expected/reference/Jobs/pi/active_person_count/aggregate-Month-reference.csv b/...sion/current/expected/reference/Jobs/pi/active_person_count/aggregate-Month-reference.csv
@@ -19,7 +19,7 @@ Fulmar,2
 "Shearwater, Balearic",2
 "Warbler, Cetti's",2
 "Warbler, Hooded",2
-"Accentor, Alpine",1
+"Accentor, Alpiné",1
 Bittern,1
 Blackbird,1
 "Bluetail, Red-flanked",1

diff --git a/...on/current/expected/reference/Jobs/pi/active_person_count/aggregate-Quarter-reference.csv b/...on/current/expected/reference/Jobs/pi/active_person_count/aggregate-Quarter-reference.csv
@@ -19,7 +19,7 @@ Fulmar,2
 "Shearwater, Balearic",2
 "Warbler, Cetti's",2
 "Warbler, Hooded",2
-"Accentor, Alpine",1
+"Accentor, Alpiné",1
 Bittern,1
 Blackbird,1
 "Bluetail, Red-flanked",1

diff --git a/...ssion/current/expected/reference/Jobs/pi/active_person_count/aggregate-Year-reference.csv b/...ssion/current/expected/reference/Jobs/pi/active_person_count/aggregate-Year-reference.csv
@@ -19,7 +19,7 @@ Fulmar,2
 "Shearwater, Balearic",2
 "Warbler, Cetti's",2
 "Warbler, Hooded",2
-"Accentor, Alpine",1
+"Accentor, Alpiné",1
 Bittern,1
 Blackbird,1
 "Bluetail, Red-flanked",1

diff --git a/...ssion/current/expected/reference/Jobs/pi/active_person_count/timeseries-Day-reference.csv b/...ssion/current/expected/reference/Jobs/pi/active_person_count/timeseries-Day-reference.csv
@@ -5,7 +5,7 @@ parameters
 start,end
 2016-12-22,2017-01-01
 ---------
-Day,"[Tern, Caspian] Number of Users: Active","[Fieldfare] Number of Users: Active","[Lark, Calandra] Number of Users: Active","[White, Great] Number of Users: Active","[Dowitcher, Short-billed] Number of Users: Active","[Jackdaw] Number of Users: Active","[Chaffinch] Number of Users: Active","[Flycatcher, Taiga] Number of Users: Active","[Fulmar] Number of Users: Active","[Scaup, Lesser] Number of Users: Active","[Shearwater, Balearic] Number of Users: Active","[Warbler, Cetti's] Number of Users: Active","[Warbler, Hooded] Number of Users: Active","[Accentor, Alpine] Number of Users: Active","[Bittern] Number of Users: Active","[Blackbird] Number of Users: Active","[Bluetail, Red-flanked] Number of Users: Active","[Bufflehead] Number of Users: Active","[Bunting, Ortolan] Number of Users: Active","[Chiffchaff, Iberian] Number of Users: Active","[Dotterel] Number of Users: Active","[Dunlin] Number of Users: Active","[Egret, Snowy] Number of Users: Active","[Goose, Red-breasted] Number of Users: Active","[Gull, Glaucous-winged] Number of Users: Active","[Gull, Yellow-legged] Number of Users: Active","[Harrier, Hen] Number of Users: Active","[Kestrel, Lesser] Number of Users: Active","[Martin, Sand] Number of Users: Active","[Moorhen] Number of Users: Active","[Nuthatch] Number of Users: Active","[Petrel, Fea's] Number of Users: Active","[Shearwater, Great] Number of Users: Active","[Shearwater, Macaronesian] Number of Users: Active","[Smew] Number of Users: Active","[Spotted, Great] Number of Users: Active","[Swift, Little] Number of Users: Active","[Thrush, Hermit] Number of Users: Active","[Warbler, Blackpoll] Number of Users: Active","[Warbler, Savi's] Number of Users: Active","[Yellowthroat, Common] Number of Users: Active"
+Day,"[Tern, Caspian] Number of Users: Active","[Fieldfare] Number of Users: Active","[Lark, Calandra] Number of Users: Active","[White, Great] Number of Users: Active","[Dowitcher, Short-billed] Number of Users: Active","[Jackdaw] Number of Users: Active","[Chaffinch] Number of Users: Active","[Flycatcher, Taiga] Number of Users: Active","[Fulmar] Number of Users: Active","[Scaup, Lesser] Number of Users: Active","[Shearwater, Balearic] Number of Users: Active","[Warbler, Cetti's] Number of Users: Active","[Warbler, Hooded] Number of Users: Active","[Accentor, Alpiné] Number of Users: Active","[Bittern] Number of Users: Active","[Blackbird] Number of Users: Active","[Bluetail, Red-flanked] Number of Users: Active","[Bufflehead] Number of Users: Active","[Bunting, Ortolan] Number of Users: Active","[Chiffchaff, Iberian] Number of Users: Active","[Dotterel] Number of Users: Active","[Dunlin] Number of Users: Active","[Egret, Snowy] Number of Users: Active","[Goose, Red-breasted] Number of Users: Active","[Gull, Glaucous-winged] Number of Users: Active","[Gull, Yellow-legged] Number of Users: Active","[Harrier, Hen] Number of Users: Active","[Kestrel, Lesser] Number of Users: Active","[Martin, Sand] Number of Users: Active","[Moorhen] Number of Users: Active","[Nuthatch] Number of Users: Active","[Petrel, Fea's] Number of Users: Active","[Shearwater, Great] Number of Users: Active","[Shearwater, Macaronesian] Number of Users: Active","[Smew] Number of Users: Active","[Spotted, Great] Number of Users: Active","[Swift, Little] Number of Users: Active","[Thrush, Hermit] Number of Users: Active","[Warbler, Blackpoll] Number of Users: Active","[Warbler, Savi's] Number of Users: Active","[Yellowthroat, Common] Number of Users: Active"
 2016-12-22,0,0,1,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0
 2016-12-23,1,0,1,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0
 2016-12-24,1,0,1,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0

diff --git a/...ion/current/expected/reference/Jobs/pi/active_person_count/timeseries-Month-reference.csv b/...ion/current/expected/reference/Jobs/pi/active_person_count/timeseries-Month-reference.csv
@@ -5,7 +5,7 @@ parameters
 start,end
 2016-12-22,2017-01-01
 ---------
-Month,"[Tern, Caspian] Number of Users: Active","[Fieldfare] Number of Users: Active","[Lark, Calandra] Number of Users: Active","[White, Great] Number of Users: Active","[Dowitcher, Short-billed] Number of Users: Active","[Jackdaw] Number of Users: Active","[Chaffinch] Number of Users: Active","[Flycatcher, Taiga] Number of Users: Active","[Fulmar] Number of Users: Active","[Scaup, Lesser] Number of Users: Active","[Shearwater, Balearic] Number of Users: Active","[Warbler, Cetti's] Number of Users: Active","[Warbler, Hooded] Number of Users: Active","[Accentor, Alpine] Number of Users: Active","[Bittern] Number of Users: Active","[Blackbird] Number of Users: Active","[Bluetail, Red-flanked] Number of Users: Active","[Bufflehead] Number of Users: Active","[Bunting, Ortolan] Number of Users: Active","[Chiffchaff, Iberian] Number of Users: Active","[Dotterel] Number of Users: Active","[Dunlin] Number of Users: Active","[Egret, Snowy] Number of Users: Active","[Goose, Red-breasted] Number of Users: Active","[Gull, Glaucous-winged] Number of Users: Active","[Gull, Yellow-legged] Number of Users: Active","[Harrier, Hen] Number of Users: Active","[Kestrel, Lesser] Number of Users: Active","[Martin, Sand] Number of Users: Active","[Moorhen] Number of Users: Active","[Nuthatch] Number of Users: Active","[Petrel, Fea's] Number of Users: Active","[Shearwater, Great] Number of Users: Active","[Shearwater, Macaronesian] Number of Users: Active","[Smew] Number of Users: Active","[Spotted, Great] Number of Users: Active","[Swift, Little] Number of Users: Active","[Thrush, Hermit] Number of Users: Active","[Warbler, Blackpoll] Number of Users: Active","[Warbler, Savi's] Number of Users: Active","[Yellowthroat, Common] Number of Users: Active"
+Month,"[Tern, Caspian] Number of Users: Active","[Fieldfare] Number of Users: Active","[Lark, Calandra] Number of Users: Active","[White, Great] Number of Users: Active","[Dowitcher, Short-billed] Number of Users: Active","[Jackdaw] Number of Users: Active","[Chaffinch] Number of Users: Active","[Flycatcher, Taiga] Number of Users: Active","[Fulmar] Number of Users: Active","[Scaup, Lesser] Number of Users: Active","[Shearwater, Balearic] Number of Users: Active","[Warbler, Cetti's] Number of Users: Active","[Warbler, Hooded] Number of Users: Active","[Accentor, Alpiné] Number of Users: Active","[Bittern] Number of Users: Active","[Blackbird] Number of Users: Active","[Bluetail, Red-flanked] Number of Users: Active","[Bufflehead] Number of Users: Active","[Bunting, Ortolan] Number of Users: Active","[Chiffchaff, Iberian] Number of Users: Active","[Dotterel] Number of Users: Active","[Dunlin] Number of Users: Active","[Egret, Snowy] Number of Users: Active","[Goose, Red-breasted] Number of Users: Active","[Gull, Glaucous-winged] Number of Users: Active","[Gull, Yellow-legged] Number of Users: Active","[Harrier, Hen] Number of Users: Active","[Kestrel, Lesser] Number of Users: Active","[Martin, Sand] Number of Users: Active","[Moorhen] Number of Users: Active","[Nuthatch] Number of Users: Active","[Petrel, Fea's] Number of Users: Active","[Shearwater, Great] Number of Users: Active","[Shearwater, Macaronesian] Number of Users: Active","[Smew] Number of Users: Active","[Spotted, Great] Number of Users: Active","[Swift, Little] Number of Users: Active","[Thrush, Hermit] Number of Users: Active","[Warbler, Blackpoll] Number of Users: Active","[Warbler, Savi's] Number of Users: Active","[Yellowthroat, Common] Number of Users: Active"
 2016-12,6,4,4,4,3,3,2,2,2,2,2,2,2,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1
 2017-01,2,3,4,3,3,2,2,2,1,1,0,0,1,1,1,0,1,1,1,1,0,1,1,1,0,1,0,1,1,1,0,0,0,0,0,0,1,0,0,0,0
 ---------
diff --git a/...n/current/expected/reference/Jobs/pi/active_person_count/timeseries-Quarter-reference.csv b/...n/current/expected/reference/Jobs/pi/active_person_count/timeseries-Quarter-reference.csv
@@ -5,7 +5,7 @@ parameters
 start,end
 2016-12-22,2017-01-01
 ---------
-Quarter,"[Tern, Caspian] Number of Users: Active","[Fieldfare] Number of Users: Active","[Lark, Calandra] Number of Users: Active","[White, Great] Number of Users: Active","[Dowitcher, Short-billed] Number of Users: Active","[Jackdaw] Number of Users: Active","[Chaffinch] Number of Users: Active","[Flycatcher, Taiga] Number of Users: Active","[Fulmar] Number of Users: Active","[Scaup, Lesser] Number of Users: Active","[Shearwater, Balearic] Number of Users: Active","[Warbler, Cetti's] Number of Users: Active","[Warbler, Hooded] Number of Users: Active","[Accentor, Alpine] Number of Users: Active","[Bittern] Number of Users: Active","[Blackbird] Number of Users: Active","[Bluetail, Red-flanked] Number of Users: Active","[Bufflehead] Number of Users: Active","[Bunting, Ortolan] Number of Users: Active","[Chiffchaff, Iberian] Number of Users: Active","[Dotterel] Number of Users: Active","[Dunlin] Number of Users: Active","[Egret, Snowy] Number of Users: Active","[Goose, Red-breasted] Number of Users: Active","[Gull, Glaucous-winged] Number of Users: Active","[Gull, Yellow-legged] Number of Users: Active","[Harrier, Hen] Number of Users: Active","[Kestrel, Lesser] Number of Users: Active","[Martin, Sand] Number of Users: Active","[Moorhen] Number of Users: Active","[Nuthatch] Number of Users: Active","[Petrel, Fea's] Number of Users: Active","[Shearwater, Great] Number of Users: Active","[Shearwater, Macaronesian] Number of Users: Active","[Smew] Number of Users: Active","[Spotted, Great] Number of Users: Active","[Swift, Little] Number of Users: Active","[Thrush, Hermit] Number of Users: Active","[Warbler, Blackpoll] Number of Users: Active","[Warbler, Savi's] Number of Users: Active","[Yellowthroat, Common] Number of Users: Active"
+Quarter,"[Tern, Caspian] Number of Users: Active","[Fieldfare] Number of Users: Active","[Lark, Calandra] Number of Users: Active","[White, Great] Number of Users: Active","[Dowitcher, Short-billed] Number of Users: Active","[Jackdaw] Number of Users: Active","[Chaffinch] Number of Users: Active","[Flycatcher, Taiga] Number of Users: Active","[Fulmar] Number of Users: Active","[Scaup, Lesser] Number of Users: Active","[Shearwater, Balearic] Number of Users: Active","[Warbler, Cetti's] Number of Users: Active","[Warbler, Hooded] Number of Users: Active","[Accentor, Alpiné] Number of Users: Active","[Bittern] Number of Users: Active","[Blackbird] Number of Users: Active","[Bluetail, Red-flanked] Number of Users: Active","[Bufflehead] Number of Users: Active","[Bunting, Ortolan] Number of Users: Active","[Chiffchaff, Iberian] Number of Users: Active","[Dotterel] Number of Users: Active","[Dunlin] Number of Users: Active","[Egret, Snowy] Number of Users: Active","[Goose, Red-breasted] Number of Users: Active","[Gull, Glaucous-winged] Number of Users: Active","[Gull, Yellow-legged] Number of Users: Active","[Harrier, Hen] Number of Users: Active","[Kestrel, Lesser] Number of Users: Active","[Martin, Sand] Number of Users: Active","[Moorhen] Number of Users: Active","[Nuthatch] Number of Users: Active","[Petrel, Fea's] Number of Users: Active","[Shearwater, Great] Number of Users: Active","[Shearwater, Macaronesian] Number of Users: Active","[Smew] Number of Users: Active","[Spotted, Great] Number of Users: Active","[Swift, Little] Number of Users: Active","[Thrush, Hermit] Number of Users: Active","[Warbler, Blackpoll] Number of Users: Active","[Warbler, Savi's] Number of Users: Active","[Yellowthroat, Common] Number of Users: Active"
 "2016 Q4",6,4,4,4,3,3,2,2,2,2,2,2,2,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1
 "2017 Q1",2,3,4,3,3,2,2,2,1,1,0,0,1,1,1,0,1,1,1,1,0,1,1,1,0,1,0,1,1,1,0,0,0,0,0,0,1,0,0,0,0
 ---------