finalising e2e

cypress/support/services/ldes-server.ts

@@ -17,7 +17,7 @@ export class LdesServer implements CanCheckAvailability {
     private isReady(containerId: string, message?: string, minOccurences: number = 1 ) {
         return cy.exec(`docker logs ${containerId}`)
             .then(result => {
-                const regex = new RegExp(message || LdesServer.DatabaseUpgradeFinished, "g");
+                const regex = new RegExp(LdesServer.ApplicationStarted || LdesServer.DatabaseUpgradeFinished, "g");
                 return (result.stdout.match(regex) || []).length >= minOccurences;
             });
     }

cypress/support/services/postgres-rest-api.ts

@@ -0,0 +1,45 @@
+/// <reference types="cypress" />
+
+import { timeouts } from "../common";
+
+type CountResult = { count: number, ids: string[] };
+
+type FragmentInfo = { _id: string, nrOfMembersAdded: number }
+type DocumentResult = { count: number, documents: FragmentInfo[] };
+
+export class PostgresRestApi {
+
+    constructor(public baseUrl: string) { }
+
+    checkCount(collection: string, count: number, checkFn: (actual: number, expected: number) => boolean = (x, y) => x === y) {
+        return cy.waitUntil(() => this.hasCount(collection, count, checkFn), { timeout: timeouts.slowAction, interval: timeouts.slowCheck, errorMsg: `Timed out waiting for document collection '${collection}' to correctly compare to ${count}` });
+    }
+
+    checkFragmentMemberCount(fragment: string, count: number, checkFn: (actual: number, expected: number) => boolean = (x, y) => x === y) {
+        return cy.waitUntil(() => this.hasFragmentMemberCount(fragment, count, checkFn), { timeout: timeouts.slowAction, interval: timeouts.slowCheck, errorMsg: `Timed out waiting for fragment member count of '${fragment}' to correctly compare to ${count}` });
+    }
+
+    private hasFragmentMemberCount(fragment: string, count: number, checkFn: (actual: number, expected: number) => boolean) {
+        return cy.request(`${this.baseUrl}/fragmentation_fragment?includeDocuments=true`)
+        .then(response => response.body)
+        .then((body: DocumentResult) => body.documents.find(x => x._id === fragment))
+        .then((fragment: FragmentInfo) => cy.log('Actual fragment member count: ' + fragment.nrOfMembersAdded).then(() => checkFn(fragment.nrOfMembersAdded , count)));
+    }
+
+    private hasCount(collection: string, count: number, checkFn: (actual: number, expected: number) => boolean) {
+        return cy.request(`${this.baseUrl}/${collection}`)
+            .then(response => response.body)
+            .then((result: CountResult) => cy.log('Actual count: ' + result.count).then(() => checkFn(result.count , count)));
+    }
+
+    count(collection: string) {
+        return cy.request(`${this.baseUrl}/${collection}`)
+            .then(response => response.body)
+            .then((result: CountResult) => result.count);
+    }
+
+    private documentIds(document: string) {
+        return cy.request(`${this.baseUrl}/${document}?includeIds=true`)
+                 .then(response => response.body && (response.body.ids as string[]));
+    }
+}

cypress/support/step_definitions/common_step_definitions.ts

@@ -13,6 +13,7 @@ import {Gtfs2Ldes} from "../services/gtfs2ldes";
 import {Fragment} from "../ldes";
 import {LdesClientWorkbench} from "../services/ldes-client-workbench";
 import {LdiLdesDiscoverer} from "../services/ldi-ldes-discoverer";
+import {PostgresRestApi} from "../services/postgres-rest-api";
 
 let testContext: any;
 const ldesMemberCollection = 'ingest_ldesmember';
@@ -24,7 +25,7 @@ export const workbenchLdio = new LdesWorkbenchLdio('http://localhost:8081');
 export const clientWorkbench = new LdesClientWorkbench('http://localhost:8081');
 export const sink = new TestMessageSink('http://localhost:9003');
 export const simulator = new LdesServerSimulator('http://localhost:9011');
-export const mongo = new MongoRestApi('http://localhost:9019');
+export const postgres = new PostgresRestApi('http://localhost:9018');
 export const jsonDataGenerator = new TestMessageGenerator();
 export const server = new LdesServer('http://localhost:8080');
 export const gtfs2ldes = new Gtfs2Ldes();
@@ -55,10 +56,6 @@ export function testPartialPath() {
     return testContext && testContext.testPartialPath;
 }
 
-export function testDatabase() {
-    return testContext && testContext.database;
-}
-
 export function ensureRelationCount(fragment: Fragment, amount: number) {
     return waitForFragment(fragment, x => x.relations.length >= amount, `have at least ${amount} relations`);
 }
@@ -278,23 +275,23 @@ When('I start the JSON Data Generator', () => {
 })
 
 When('the LDES contains {int} members', (count: number) => {
-    mongo.checkCount(testContext.database, ldesMemberCollection, count);
+    postgres.checkCount(ldesMemberCollection, count);
 })
 
 When('the LDES contains {int} fragments', (count: number) => {
-    mongo.checkCount(testContext.database, ldesFragmentCollection, count);
+    postgres.checkCount(ldesFragmentCollection, count);
 })
 
 When('the LDES contains at least {int} members', (count: number) => {
-    mongo.checkCount(testContext.database, ldesMemberCollection, count, (x, y) => x >= y);
+    postgres.checkCount(ldesMemberCollection, count, (x, y) => x >= y);
 })
 
 When('the LDES fragment {string} contains at least {int} members', (fragmentId: string, count: number) => {
-    mongo.checkFragmentMemberCount(testContext.database, fragmentId, count, (x, y) => x >= y);
+    postgres.checkFragmentMemberCount(fragmentId, count, (x, y) => x >= y);
 })
 
 When('the LDES contains at least {int} fragments', (count: number) => {
-    mongo.checkCount(testContext.database, ldesFragmentCollection, count, (x, y) => x >= y);
+    postgres.checkCount(ldesFragmentCollection, count, (x, y) => x >= y);
 })
 
 export function waitUntilMemberCountStable() {
@@ -305,7 +302,7 @@ export function waitUntilMemberCountStable() {
         {
             timeout: timeouts.fastAction,
             interval: timeouts.check,
-            errorMsg: `Timed out waiting for database '${testContext.database}' ingest count to retain the same (last: ${previousCount})`
+            errorMsg: `Timed out waiting for ingest count to retain the same (last: ${previousCount})`
         }
     );
 }
@@ -342,7 +339,7 @@ Then('the sink contains {int} members in collection {string}', (count: number, c
 })
 
 export function currentMemberCount() {
-    return mongo.count(testContext.database, ldesMemberCollection);
+    return postgres.count(ldesMemberCollection);
 }
 
 Then('the LDES should contain {int} members', (memberCount: number) => {
@@ -351,7 +348,7 @@ Then('the LDES should contain {int} members', (memberCount: number) => {
 
 Then('the LDES member count increases', () => {
     currentMemberCount().then(currentCount =>
-        mongo.checkCount(testContext.database, ldesMemberCollection, currentCount,
+        postgres.checkCount(ldesMemberCollection, currentCount,
             (actual, expected) => actual > expected));
 })
 

cypress/support/step_definitions/server.upgrading.feature.ts

@@ -1,5 +1,5 @@
 import { Given, When, Then } from "@badeball/cypress-cucumber-preprocessor";
-import { createAndStartService, stopAndRemoveService, mongo, testDatabase, waitUntilMemberCountStable } from "./common_step_definitions";
+import { createAndStartService, stopAndRemoveService, postgres, waitUntilMemberCountStable } from "./common_step_definitions";
 import { LdesServer } from "../services";
 import {EventStream} from "../ldes";
 
@@ -56,7 +56,7 @@ Given('the old ldesmember collection is structured as expected', () => {
 })
 
 When('the LDES contains at least {int} members in the old database', (count: number) => {
-    mongo.checkCount(testDatabase(), 'ldesmember', count, (x, y) => x >= y);
+    postgres.checkCount('ldesmember', count, (x, y) => x >= y);
 })
 
 Given('the old LDES server is available', () => {

tests/007.server-ingest-large-ldes/config/ldes-server.config.yml

@@ -9,8 +9,10 @@ management:
 ldes-server:
   host-name: http://localhost:8080
 spring:
-  data:
-    mongodb:
-      database: gtfs
-      host: ldes-mongodb
-      port: 27017
+  datasource:
+    url: jdbc:postgresql://ldes-postgres:5432/test
+    username: admin
+    password: admin
+  jpa:
+    hibernate:
+      ddl-auto: update

tests/007.server-ingest-large-ldes/docker-compose.yml

@@ -62,24 +62,36 @@ services:
     networks:
       - ldes
     depends_on:
-      - ldes-mongodb
+      - ldes-postgres
 
-  ldes-mongodb:
-    container_name: ${USECASE_NAME:-gtfs-ingest-ldes}_ldes-mongodb
-    image: mongo:${MONGODB_TAG:-latest}
+  ldes-postgres:
+    container_name: ldes-postgres
+    image: postgres:14-alpine
     ports:
-      - ${MONGODB_PORT:-27017}:27017
+      - "5432:5432"
+    environment:
+      - POSTGRES_PASSWORD=admin
+      - POSTGRES_USER=admin
+      - POSTGRES_DB=test
     networks:
       - ldes
 
-  mongodb-rest-api:
-    container_name: ${USECASE_NAME:-gtfs-ingest-ldes}_mongodb-rest-api
-    image: ghcr.io/informatievlaanderen/mongodb-rest-api:${MONGODB_REST_API_TAG:-latest}
-    environment:
-      - SILENT=false
-      - CONNECTION_URI=mongodb://ldes-mongodb:27017
+  postgrest:
+    container_name: postgres-rest-api
+    image: postgrest/postgrest:latest
     ports:
-      - ${MONGODB_REST_API_PORT:-9019}:80
+      - "9018:3000"
+    # Available environment variables documented here:
+    # https://postgrest.org/en/latest/configuration.html#environment-variables
+    environment:
+      # The standard connection URI format, documented at
+      # https://www.postgresql.org/docs/current/static/libpq-connect.html#LIBPQ-CONNSTRING
+      - PGRST_DB_URI=postgres://admin:admin@ldes-postgres:5432/test
+      - PGRST_DB_ANON_ROLE=admin
+      - PGRST_OPENAPI_SERVER_PROXY_URI=http://localhost:9018
+
+    depends_on:
+      - ldes-postgres
     networks:
       - ldes
 

tests/013.server-perform-fast-enough/config/ldes-server.config.yml

@@ -9,8 +9,10 @@ management:
 ldes-server:
   host-name: http://localhost:8080
 spring:
-  data:
-    mongodb:
-      database: gtfs-performance
-      host: ldes-mongodb
-      port: 27017
+  datasource:
+    url: jdbc:postgresql://ldes-postgres:5432/test
+    username: admin
+    password: admin
+  jpa:
+    hibernate:
+      ddl-auto: update

tests/013.server-perform-fast-enough/docker-compose.yml

@@ -36,7 +36,6 @@ services:
     image: ${LDES_SERVER:-ghcr.io/informatievlaanderen/ldes-server}:${LDES_SERVER_TAG:-latest}
     environment:
       - SIS_DATA=/tmp
-#      - SPRING_DATA_MONGODB_AUTO-INDEX-CREATION=true
     volumes:
       - ./config/ldes-server.config.yml:/application.yml:ro
       - ../../data/epsg:/tmp/Databases:rw
@@ -53,40 +52,34 @@ services:
     networks:
       - ldes
     depends_on:
-      - ldes-mongodb
+      - ldes-postgres
 
-  ldes-mongodb:
-    container_name: ${USECASE_NAME:-gtfs-direct-connect}_ldes-mongodb
-    image: mongo:${MONGODB_TAG:-latest}
+  ldes-postgres:
+    container_name: ldes-postgres
+    image: postgres:14-alpine
     ports:
-      - ${MONGODB_PORT:-27017}:27017
-    deploy:
-      resources:
-        limits:
-          cpus: '0.6'
-          memory: '1.5GB'
-        reservations:
-          cpus: '0.3'
-          memory: '768MB'
+      - "5432:5432"
+    environment:
+      - POSTGRES_PASSWORD=admin
+      - POSTGRES_USER=admin
+      - POSTGRES_DB=test
     networks:
       - ldes
 
-  mongodb-rest-api:
-    container_name: ${USECASE_NAME:-gtfs-direct-connect}_mongodb-rest-api
-    image: ghcr.io/informatievlaanderen/mongodb-rest-api:${MONGODB_REST_API_TAG:-latest}
-    environment:
-      - SILENT=false
-      - CONNECTION_URI=mongodb://ldes-mongodb:27017
-    deploy:
-      resources:
-        limits:
-          cpus: '0.1'
-          memory: '128MB'
-        reservations:
-          cpus: '0.1'
-          memory: '128MB'
+  postgrest:
+    container_name: postgres-rest-api
+    image: postgrest/postgrest:latest
     ports:
-      - ${MONGODB_REST_API_PORT:-9019}:80
+      - "9018:3000"
+    # Available environment variables documented here:
+    # https://postgrest.org/en/latest/configuration.html#environment-variables
+    environment:
+      - PGRST_DB_URI=postgres://admin:admin@ldes-postgres:5432/test
+      - PGRST_DB_ANON_ROLE=admin
+      - PGRST_OPENAPI_SERVER_PROXY_URI=http://localhost:9018
+
+    depends_on:
+      - ldes-postgres
     networks:
       - ldes