migrate server to postgres

Informatievlaanderen · May 22, 2024 · c813478 · c813478
1 parent c1370cd
commit c813478
Show file tree

Hide file tree

Showing 28 changed files with 455 additions and 224 deletions.
diff --git a/cypress/support/services/mongo-rest-api.ts b/cypress/support/services/mongo-rest-api.ts
@@ -37,17 +37,4 @@ export class MongoRestApi {
             .then(response => response.body)
             .then((result: CountResult) => result.count);
     }
-
-    private documentIds(database: string, document: string) {
-        return cy.request(`${this.baseUrl}/${database}/${document}?includeIds=true`)
-                 .then(response => response.body && (response.body.ids as string[]));
-    }
-
-    fragmentIds(database: string) {
-        return this.documentIds(database, 'fragmentation_fragment');
-    }
-
-    snapshotIds(database: string) {
-        return this.documentIds(database, 'snapshot');
-    }
 }
diff --git a/cypress/support/services/postgres-rest-api.ts b/cypress/support/services/postgres-rest-api.ts
@@ -2,13 +2,11 @@
 
 import { timeouts } from "../common";
 
-type CountResult = { count: number, ids: string[] };
-
 type FragmentInfo = { _id: string, nrOfMembersAdded: number }
 type DocumentResult = { count: number, documents: FragmentInfo[] };
 
 export class PostgresRestApi {
-    
+
     constructor(public baseUrl: string) { }
 
     checkCount(collection: string, count: number, checkFn: (actual: number, expected: number) => boolean = (x, y) => x === y) {
@@ -28,18 +26,11 @@ export class PostgresRestApi {
 
     private hasCount(collection: string, count: number, checkFn: (actual: number, expected: number) => boolean) {
         return cy.request(`${this.baseUrl}/${collection}`)
-            .then(response => response.body)
-            .then((result: CountResult) => cy.log('Actual count: ' + result.count).then(() => checkFn(result.count , count)));
+            .then((response => cy.log('Actual count: ' + response.body.length).then(() => checkFn(response.body.length , count))));
     }
 
     count(collection: string) {
         return cy.request(`${this.baseUrl}/${collection}`)
-            .then(response => response.body)
-            .then((result: CountResult) => result.count);
-    }
-
-    private documentIds(document: string) {
-        return cy.request(`${this.baseUrl}/${document}?includeIds=true`)
-                 .then(response => response.body && (response.body.ids as string[]));
+            .then(response => response.body.length);
     }
 }
diff --git a/tests/014.workbench-ngsi-v2-to-ngsi-ld/config/ldes-server.config.yml b/tests/014.workbench-ngsi-v2-to-ngsi-ld/config/ldes-server.config.yml
@@ -9,8 +9,10 @@ management:
 ldes-server:
   host-name: http://localhost:8080
 spring:
-  data:
-    mongodb:
-      database: iow
-      host: ldes-mongodb
-      port: 27017
+  datasource:
+    url: jdbc:postgresql://ldes-postgres:5432/test
+    username: admin
+    password: admin
+  jpa:
+    hibernate:
+      ddl-auto: update
diff --git a/tests/014.workbench-ngsi-v2-to-ngsi-ld/docker-compose.yml b/tests/014.workbench-ngsi-v2-to-ngsi-ld/docker-compose.yml
@@ -49,14 +49,6 @@ services:
     profiles:
       - delay-started
 
-  ldes-mongodb:
-    container_name: ${USECASE_NAME:-iow-context}_ldes-mongodb
-    image: mongo:${MONGODB_TAG:-latest}
-    ports:
-      - ${MONGODB_PORT:-27017}:27017
-    networks:
-      - ldes
-
   ldes-server:
     container_name: ${USECASE_NAME:-iow-context}_ldes-server
     image: ${LDES_SERVER:-ghcr.io/informatievlaanderen/ldes-server}:${LDES_SERVER_TAG:-latest}
@@ -70,16 +62,38 @@ services:
     networks:
       - ldes
     depends_on:
-      - ldes-mongodb
+      - ldes-postgres
 
-  mongodb-rest-api:
-    container_name: ${USECASE_NAME:-iow-context}_mongodb-rest-api
-    image: ghcr.io/informatievlaanderen/mongodb-rest-api:${MONGODB_REST_API_TAG:-latest}
+  ldes-postgres:
+    container_name: ldes-postgres
+    image: postgres:14-alpine
+    ports:
+      - "5432:5432"
     environment:
-      - SILENT=false
-      - CONNECTION_URI=mongodb://ldes-mongodb:27017
+      - POSTGRES_PASSWORD=admin
+      - POSTGRES_USER=admin
+      - POSTGRES_DB=test
+    healthcheck:
+      test: [ "CMD-SHELL", "pg_isready -U admin -d test" ]
+      interval: 5s
+      timeout: 5s
+      retries: 5
+    restart: always
+    networks:
+      - ldes
+
+  postgrest:
+    container_name: postgres-rest-api
+    image: postgrest/postgrest:latest
     ports:
-      - ${MONGODB_REST_API_PORT:-9019}:80
+      - "9018:3000"
+    environment:
+      - PGRST_DB_URI=postgres://admin:admin@ldes-postgres:5432/test
+      - PGRST_DB_ANON_ROLE=admin
+      - PGRST_OPENAPI_SERVER_PROXY_URI=http://localhost:9018
+
+    depends_on:
+      - ldes-postgres
     networks:
       - ldes
 

diff --git a/tests/015.nifi-workbench-ngsi-v2-to-oslo/config/ldes-server.config.yml b/tests/015.nifi-workbench-ngsi-v2-to-oslo/config/ldes-server.config.yml
@@ -9,8 +9,10 @@ management:
 ldes-server:
   host-name: http://localhost:8080
 spring:
-  data:
-    mongodb:
-      database: iow
-      host: ldes-mongodb
-      port: 27017
+  datasource:
+    url: jdbc:postgresql://ldes-postgres:5432/test
+    username: admin
+    password: admin
+  jpa:
+    hibernate:
+      ddl-auto: update
diff --git a/tests/015.nifi-workbench-ngsi-v2-to-oslo/docker-compose.yml b/tests/015.nifi-workbench-ngsi-v2-to-oslo/docker-compose.yml
@@ -32,14 +32,6 @@ services:
     networks:
       - ldes
 
-  ldes-mongodb:
-    container_name: ${USECASE_NAME:-iow-oslo-compliancy}_ldes-mongodb
-    image: mongo:${MONGODB_TAG:-latest}
-    ports:
-      - ${MONGODB_PORT:-27017}:27017
-    networks:
-      - ldes
-
   ldes-server:
     container_name: ${USECASE_NAME:-iow-oslo-compliancy}_ldes-server
     image: ${LDES_SERVER:-ghcr.io/informatievlaanderen/ldes-server}:${LDES_SERVER_TAG:-latest}
@@ -53,18 +45,40 @@ services:
     networks:
       - ldes
     depends_on:
-      - ldes-mongodb
+      - ldes-postgres
 
-  mongodb-rest-api:
-    container_name: ${USECASE_NAME:-iow-oslo-compliancy}_mongodb-rest-api
-    image: ghcr.io/informatievlaanderen/mongodb-rest-api:${MONGODB_REST_API_TAG:-latest}
+  ldes-postgres:
+    container_name: ldes-postgres
+    image: postgres:14-alpine
+    ports:
+      - "5432:5432"
     environment:
-      - SILENT=false
-      - CONNECTION_URI=mongodb://ldes-mongodb:27017
+      - POSTGRES_PASSWORD=admin
+      - POSTGRES_USER=admin
+      - POSTGRES_DB=test
+    healthcheck:
+      test: [ "CMD-SHELL", "pg_isready -U admin -d test" ]
+      interval: 5s
+      timeout: 5s
+      retries: 5
+    restart: always
+    networks:
+      - ldes
+
+  postgrest:
+    container_name: postgres-rest-api
+    image: postgrest/postgrest:latest
     ports:
-      - ${MONGODB_REST_API_PORT:-9019}:80
+      - "9018:3000"
+    environment:
+      - PGRST_DB_URI=postgres://admin:admin@ldes-postgres:5432/test
+      - PGRST_DB_ANON_ROLE=admin
+      - PGRST_OPENAPI_SERVER_PROXY_URI=http://localhost:9018
+
+    depends_on:
+      - ldes-postgres
     networks:
-      - ldes    
+      - ldes
 
 networks:
   ldes:

diff --git a/tests/016.mixed-workbench-ngsi-v2-to-oslo/config/ldes-server.config.yml b/tests/016.mixed-workbench-ngsi-v2-to-oslo/config/ldes-server.config.yml
@@ -9,8 +9,11 @@ management:
 ldes-server:
   host-name: http://localhost:8080
 spring:
-  data:
-    mongodb:
-      database: iow
-      host: ldes-mongodb
-      port: 27017
+  datasource:
+    url: jdbc:postgresql://ldes-postgres:5432/test
+    username: admin
+    password: admin
+  jpa:
+    hibernate:
+      ddl-auto: update
+
diff --git a/tests/016.mixed-workbench-ngsi-v2-to-oslo/docker-compose.yml b/tests/016.mixed-workbench-ngsi-v2-to-oslo/docker-compose.yml
@@ -32,14 +32,6 @@ services:
     networks:
       - ldes
 
-  ldes-mongodb:
-    container_name: ${USECASE_NAME:-iow-use-ldio}_ldes-mongodb
-    image: mongo:${MONGODB_TAG:-latest}
-    ports:
-      - ${MONGODB_PORT:-27017}:27017
-    networks:
-      - ldes
-
   ldes-server:
     container_name: ${USECASE_NAME:-iow-use-ldio}_ldes-server
     image: ${LDES_SERVER:-ghcr.io/informatievlaanderen/ldes-server}:${LDES_SERVER_TAG:-latest}
@@ -64,14 +56,35 @@ services:
     networks:
       - ldes
 
-  mongodb-rest-api:
-    container_name: ${USECASE_NAME:-iow-use-ldio}_mongodb-rest-api
-    image: ghcr.io/informatievlaanderen/mongodb-rest-api:${MONGODB_REST_API_TAG:-latest}
+  ldes-postgres:
+    container_name: ldes-postgres
+    image: postgres:14-alpine
+    ports:
+      - "5432:5432"
     environment:
-      - SILENT=false
-      - CONNECTION_URI=mongodb://ldes-mongodb:27017
+      - POSTGRES_PASSWORD=admin
+      - POSTGRES_USER=admin
+      - POSTGRES_DB=test
+    healthcheck:
+      test: [ "CMD-SHELL", "pg_isready -U admin -d test" ]
+      interval: 5s
+      timeout: 5s
+      retries: 5
+    restart: always
+    networks:
+      - ldes
+
+  postgrest:
+    container_name: postgres-rest-api
+    image: postgrest/postgrest:latest
     ports:
-      - ${MONGODB_REST_API_PORT:-9019}:80
+      - "9018:3000"
+    environment:
+      - PGRST_DB_URI=postgres://admin:admin@ldes-postgres:5432/test
+      - PGRST_DB_ANON_ROLE=admin
+      - PGRST_OPENAPI_SERVER_PROXY_URI=http://localhost:9018
+    depends_on:
+      - ldes-postgres
     networks:
       - ldes
 

diff --git a/tests/017.ldio-workbench-ngsi-v2-to-oslo/config/ldes-server.config.yml b/tests/017.ldio-workbench-ngsi-v2-to-oslo/config/ldes-server.config.yml
@@ -9,9 +9,12 @@ management:
 ldes-server:
   host-name: http://localhost:8080
 spring:
-  data:
-    mongodb:
-      database: iow
-      host: ldes-mongodb
-      port: 27017
+  datasource:
+    url: jdbc:postgresql://ldes-postgres:5432/test
+    username: admin
+    password: admin
+  jpa:
+    hibernate:
+      ddl-auto: update
+
 
diff --git a/tests/017.ldio-workbench-ngsi-v2-to-oslo/docker-compose.yml b/tests/017.ldio-workbench-ngsi-v2-to-oslo/docker-compose.yml
@@ -19,14 +19,6 @@ services:
     depends_on:
       - ldio-workbench
 
-  ldes-mongodb:
-    container_name: ${USECASE_NAME:-nifi-less-approach}_ldes-mongodb
-    image: mongo:${MONGODB_TAG:-latest}
-    ports:
-      - ${MONGODB_PORT:-27017}:27017
-    networks:
-      - ldes
-
   ldes-server:
     container_name: ${USECASE_NAME:-nifi-less-approach}_ldes-server
     image: ${LDES_SERVER:-ghcr.io/informatievlaanderen/ldes-server}:${LDES_SERVER_TAG:-latest}
@@ -40,7 +32,7 @@ services:
     networks:
       - ldes
     depends_on:
-      - ldes-mongodb
+      - ldes-postgres
 
   ldio-workbench:
     container_name: ${USECASE_NAME:-nifi-less-approach}_ldio-workbench
@@ -53,14 +45,36 @@ services:
     networks:
       - ldes
 
-  mongodb-rest-api:
-    container_name: ${USECASE_NAME:-nifi-less-approach}_mongodb-rest-api
-    image: ghcr.io/informatievlaanderen/mongodb-rest-api:${MONGODB_REST_API_TAG:-latest}
+  ldes-postgres:
+    container_name: ldes-postgres
+    image: postgres:14-alpine
+    ports:
+      - "5432:5432"
     environment:
-      - SILENT=false
-      - CONNECTION_URI=mongodb://ldes-mongodb:27017
+      - POSTGRES_PASSWORD=admin
+      - POSTGRES_USER=admin
+      - POSTGRES_DB=test
+    healthcheck:
+      test: [ "CMD-SHELL", "pg_isready -U admin -d test" ]
+      interval: 5s
+      timeout: 5s
+      retries: 5
+    restart: always
+    networks:
+      - ldes
+
+  postgrest:
+    container_name: postgres-rest-api
+    image: postgrest/postgrest:latest
     ports:
-      - ${MONGODB_REST_API_PORT:-9019}:80
+      - "9018:3000"
+    environment:
+      - PGRST_DB_URI=postgres://admin:admin@ldes-postgres:5432/test
+      - PGRST_DB_ANON_ROLE=admin
+      - PGRST_OPENAPI_SERVER_PROXY_URI=http://localhost:9018
+
+    depends_on:
+      - ldes-postgres
     networks:
       - ldes
 

diff --git a/tests/019.server-supports-cacheability/config/ldes-server.config.yml b/tests/019.server-supports-cacheability/config/ldes-server.config.yml
@@ -9,11 +9,13 @@ management:
 ldes-server:
   host-name: http://localhost:8080
 spring:
-  data:
-    mongodb:
-      database: Gent
-      host: ldes-mongodb
-      port: 27017
+  datasource:
+    url: jdbc:postgresql://ldes-postgres:5432/test
+    username: admin
+    password: admin
+  jpa:
+    hibernate:
+      ddl-auto: update
 rest:
   maxage: 10
   maxage-immutable: 604800