Address PR feedback

Signed-off-by: John Gomersall <[email protected]>

.env

@@ -9,6 +9,7 @@ POSTGRES_PORT=127.0.0.1:5512
 POSTGRES_DB=query
 POSTGRES_USER=productopener
 POSTGRES_PASSWORD=productopener
+POSTGRES_SHM_SIZE=256m
 COMMON_NET_NAME=po_default
 MONGO_URI=mongodb://localhost:27017
 #REDIS_URL=redis://localhost:6379

.github/workflows/container-deploy.yml

@@ -34,6 +34,7 @@ jobs:
           echo "COMMON_NET_NAME=po_webnet" >> $GITHUB_ENV
           echo "MONGO_URI=mongodb://10.1.0.200:27017" >> $GITHUB_ENV
           echo "REDIS_URL=redis://redis:6379" >> $GITHUB_ENV
+          echo "POSTGRES_SHM_SIZE=1g" >> $GITHUB_ENV
       - name: Set various variable for production deployment
         if: matrix.env == 'off-query-org'
         run: |
@@ -46,6 +47,7 @@ jobs:
           # mongodb and redis (through stunnel)
           echo "MONGO_URI=mongodb://10.1.0.102:27017" >> $GITHUB_ENV
           echo "REDIS_URL=redis://10.1.0.122:6379" >> $GITHUB_ENV
+          echo "POSTGRES_SHM_SIZE=256m" >> $GITHUB_ENV
       - name: Wait for container build workflow
         uses: tomchv/[email protected]
         id: wait-build
@@ -124,6 +126,7 @@ jobs:
             echo "COMMON_NET_NAME=${{ env.COMMON_NET_NAME }}" >> .env
             echo "MONGO_URI=${{ env.MONGO_URI }}" >> .env
             echo "REDIS_URL=${{ env.REDIS_URL }}" >> .env
+            echo "POSTGRES_SHM_SIZE=${{ env.POSTGRES_SHM_SIZE }}" >> .env
             echo "LOG_LEVEL=log" >> .env
 
 

docker-compose.yml

@@ -6,7 +6,7 @@ services:
       - POSTGRES_USER
       - POSTGRES_PASSWORD
       - POSTGRES_DB
-    shm_size: 256m
+    shm_size: ${POSTGRES_SHM_SIZE}
     volumes:
       - dbdata:/var/lib/postgresql/data
     networks:

src/domain/services/import.service.ts

@@ -241,15 +241,16 @@ export class ImportService {
     fullImport = false,
     source = ProductSource.FULL_LOAD,
   ) {
-    const autoCommit = source !== ProductSource.EVENT;
+    // Commit after each tag for bulk (non-Redis) loads to minimise server snapshot size
+    const commitPerTag = source !== ProductSource.EVENT;
 
     this.logger.debug(`Updating tags for updateId: ${updateId}`);
 
     const connection = this.em.getConnection();
 
     // Fix ingredients
     let logText = `Updated ingredients`;
-    if (autoCommit) await connection.execute('begin');
+    if (commitPerTag) await connection.execute('begin');
     const deleted = await connection.execute(
       `delete from product_ingredient 
     where product_id in (select id from product 
@@ -335,15 +336,15 @@ export class ImportService {
       affectedRows = results['affectedRows'];
       logText += ` > ${affectedRows}`;
     }
-    if (autoCommit) await connection.execute('commit');
+    if (commitPerTag) await connection.execute('commit');
     this.logger.debug(logText + ' rows');
 
     for (const [tag, entity] of Object.entries(ProductTagMap.MAPPED_TAGS)) {
       let logText = `Updated ${tag}`;
       // Get the underlying table name for the entity
       const tableName = this.em.getMetadata(entity).tableName;
 
-      if (autoCommit) await connection.execute('begin');
+      if (commitPerTag) await connection.execute('begin');
 
       // Delete existing tags for products that were imorted on this run
       const deleted = await connection.execute(
@@ -365,8 +366,7 @@ export class ImportService {
         'run',
       );
 
-      // Commit after each tag to minimise server snapshot size
-      if (autoCommit) await connection.execute('commit');
+      if (commitPerTag) await connection.execute('commit');
 
       // If this is a full load we can flag the tag as now available for query
       if (fullImport) {

src/mikro-orm.config.ts

@@ -52,6 +52,7 @@ export default defineConfig({
   },
   pool: {
     afterCreate: function (conn: any, done: any) {
+      // issue a query to verify SQL connection is working
       conn.query('select 1 as result', function (err) {
         conn.on('notice', function (msg) {
           logger.error('Notice from PostgreSQL: ' + msg.message);