fix: stream_json and migrations (#2697)

* fix: stream_json fix: deactivate datasets in etl fix: deactivate datasets in etl fix type * fix fraudcheck.labels migration --------- Co-authored-by: Jonathan Fallon <[email protected]>
betagouv · Dec 13, 2024 · 99e4fca · 99e4fca
1 parent 1cabad9
commit 99e4fca
Show file tree

Hide file tree

Showing 11 changed files with 126 additions and 71 deletions.
diff --git a/api/deno.lock b/api/deno.lock
diff --git a/api/src/db/migrations/20241122000000-alter-fraudcheck_label.sql b/api/src/db/migrations/20241122000000-alter-fraudcheck_label.sql
@@ -1 +1 @@
-ALTER TABLE FRAUDCHECK.LABELS DROP COLUMN geo_code;
+ALTER TABLE FRAUDCHECK.LABELS DROP COLUMN IF EXISTS geo_code CASCADE;
diff --git a/api/src/deps.ts b/api/src/deps.ts
@@ -50,7 +50,6 @@ export type {
   Response,
   Router,
 } from "npm:express@^4";
-//export type { FilterOptions as StreamJsonOptions } from "npm:stream-json@^1.8/filters/FilterBase.js";
 // @deno-types="npm:@types/express@^4"
 import express from "npm:express@^4";
 import extractZip from "npm:extract-zip@^2.0";
@@ -60,8 +59,6 @@ export { extractFull } from "npm:node-7z@^3.0";
 // @deno-types="npm:@types/node-7z@^2"
 export type { SevenZipOptions } from "npm:node-7z@^3";
 // @deno-types="https://cdn.sheetjs.com/xlsx-0.20.2/package/types/index.d.ts"
-//export const jsonStreamArrayUrl = "npm:stream-json@^1.8/streamers/StreamArray.js";
-//export const jsonPickUrl = "npm:stream-json@^1.8/filters/Pick.js";
 import * as xlsx from "https://cdn.sheetjs.com/xlsx-0.20.3/package/xlsx.mjs";
 import excel from "npm:exceljs@^4.4";
 

diff --git a/api/src/etl/datasets/eurostat/countries/2020/EurostatCountries2020.ts b/api/src/etl/datasets/eurostat/countries/2020/EurostatCountries2020.ts
@@ -1,17 +1,13 @@
 import { AbstractDataset } from "../../../../common/AbstractDataset.ts";
 import { streamData } from "../../../../helpers/index.ts";
-import {
-  ArchiveFileTypeEnum,
-  FileTypeEnum,
-} from "../../../../interfaces/index.ts";
+import { ArchiveFileTypeEnum, FileTypeEnum } from "../../../../interfaces/index.ts";
 
 export class EurostatCountries2020 extends AbstractDataset {
   static producer = "eurostat";
   static dataset = "countries";
   static year = 2020;
   static table = "eurostat_countries_2020";
-  static url =
-    "https://gisco-services.ec.europa.eu/distribution/v2/countries/geojson/CNTR_RG_01M_2020_4326.geojson";
+  static url = "https://gisco-services.ec.europa.eu/distribution/v2/countries/geojson/CNTR_RG_01M_2020_4326.geojson";
   readonly fileArchiveType: ArchiveFileTypeEnum = ArchiveFileTypeEnum.None;
   readonly rows: Map<string, [string, string]> = new Map([
     ["codeiso3", ["properties->>ISO3_CODE", "varchar"]],
@@ -41,7 +37,7 @@ export class EurostatCountries2020 extends AbstractDataset {
                 )
                 WITH tmp as(
                   SELECT * FROM
-                  json_to_recordset($1)
+                  json_to_recordset($1::json)
                   as tmp(type varchar, properties json,geometry json)
                 )
                 SELECT
@@ -50,7 +46,7 @@ export class EurostatCountries2020 extends AbstractDataset {
                 FROM tmp
                 ON CONFLICT DO NOTHING
               `,
-              values: [JSON.stringify(results.value.map((r: any) => r.value))],
+              values: [JSON.stringify(results.value)],
             };
             await connection.query(query);
           }

diff --git a/api/src/etl/datasets/eurostat/countries/2020/EurostatSimplifiedCountries2020.ts b/api/src/etl/datasets/eurostat/countries/2020/EurostatSimplifiedCountries2020.ts
@@ -1,17 +1,13 @@
 import { AbstractDataset } from "../../../../common/AbstractDataset.ts";
 import { streamData } from "../../../../helpers/index.ts";
-import {
-  ArchiveFileTypeEnum,
-  FileTypeEnum,
-} from "../../../../interfaces/index.ts";
+import { ArchiveFileTypeEnum, FileTypeEnum } from "../../../../interfaces/index.ts";
 
 export class EurostatSimplifiedCountries2020 extends AbstractDataset {
   static producer = "eurostat";
   static dataset = "simplified_countries";
   static year = 2020;
   static table = "eurostat_simplified_countries_2020";
-  static url =
-    "https://gisco-services.ec.europa.eu/distribution/v2/countries/geojson/CNTR_RG_60M_2020_4326.geojson";
+  static url = "https://gisco-services.ec.europa.eu/distribution/v2/countries/geojson/CNTR_RG_60M_2020_4326.geojson";
   readonly fileArchiveType: ArchiveFileTypeEnum = ArchiveFileTypeEnum.None;
   readonly rows: Map<string, [string, string]> = new Map([
     ["codeiso3", ["properties->>ISO3_CODE", "varchar"]],
@@ -41,7 +37,7 @@ export class EurostatSimplifiedCountries2020 extends AbstractDataset {
                 )
                 WITH tmp as(
                   SELECT * FROM
-                  json_to_recordset($1)
+                  json_to_recordset($1::json)
                   as tmp(type varchar, properties json,geometry json)
                 )
                 SELECT
@@ -50,7 +46,7 @@ export class EurostatSimplifiedCountries2020 extends AbstractDataset {
                 FROM tmp
                 ON CONFLICT DO NOTHING
               `,
-              values: [JSON.stringify(results.value.map((r: any) => r.value))],
+              values: [JSON.stringify(results.value)],
             };
             await connection.query(query);
           }

diff --git a/api/src/etl/datasets/eurostat/countries/2024/EurostatCountries2024.ts b/api/src/etl/datasets/eurostat/countries/2024/EurostatCountries2024.ts
@@ -1,17 +1,13 @@
 import { AbstractDataset } from "../../../../common/AbstractDataset.ts";
 import { streamData } from "../../../../helpers/index.ts";
-import {
-  ArchiveFileTypeEnum,
-  FileTypeEnum,
-} from "../../../../interfaces/index.ts";
+import { ArchiveFileTypeEnum, FileTypeEnum } from "../../../../interfaces/index.ts";
 
 export class EurostatCountries2024 extends AbstractDataset {
   static producer = "eurostat";
   static dataset = "countries";
   static year = 2024;
   static table = "eurostat_countries_2024";
-  static url =
-    "https://gisco-services.ec.europa.eu/distribution/v2/countries/geojson/CNTR_RG_01M_2024_4326.geojson";
+  static url = "https://gisco-services.ec.europa.eu/distribution/v2/countries/geojson/CNTR_RG_01M_2024_4326.geojson";
   readonly fileArchiveType: ArchiveFileTypeEnum = ArchiveFileTypeEnum.None;
   readonly rows: Map<string, [string, string]> = new Map([
     ["codeiso3", ["properties->>ISO3_CODE", "varchar"]],
@@ -41,7 +37,7 @@ export class EurostatCountries2024 extends AbstractDataset {
                 )
                 WITH tmp as(
                   SELECT * FROM
-                  json_to_recordset($1)
+                  json_to_recordset($1::json)
                   as tmp(type varchar, properties json,geometry json)
                 )
                 SELECT
@@ -50,7 +46,7 @@ export class EurostatCountries2024 extends AbstractDataset {
                 FROM tmp
                 ON CONFLICT DO NOTHING
               `,
-              values: [JSON.stringify(results.value.map((r: any) => r.value))],
+              values: [JSON.stringify(results.value)],
             };
             await connection.query(query);
           }
Original file line number	Diff line number	Diff line change
		@@ -1 +1 @@
		ALTER TABLE FRAUDCHECK.LABELS DROP COLUMN geo_code;
		ALTER TABLE FRAUDCHECK.LABELS DROP COLUMN IF EXISTS geo_code CASCADE;