feat(pub-period-migration): publication period migration on big dataset.

folio-org · Oct 20, 2024 · 1078f0f · 1078f0f
1 parent ba1a4ce
commit 1078f0f
Show file tree

Hide file tree

Showing 26 changed files with 1,009 additions and 126 deletions.
diff --git a/NEWS.md b/NEWS.md
@@ -3,8 +3,8 @@
 * Migrate "publicationPeriod" data to the Dates object and remove it from the Instance schema ([MODINVSTOR-1232](https://folio-org.atlassian.net/browse/MODINVSTOR-1232))
 
 ### New APIs versions
-* Provides `instance-storage 11.0`
-* Provides `instance-storage-batch 3.0`
+* Provides `instance-storage 11.1`
+* Provides `instance-storage-batch 3.1`
 * Provides `instance-storage-batch-sync 3.0`
 * Provides `instance-storage-batch-sync-unsafe 3.0`
 * Provides `inventory-view 3.0`
@@ -16,6 +16,7 @@
 
 ### Features
 * Add module descriptor validator plugin and fix the permission names ([MODINVSTOR-1247](https://folio-org.atlassian.net/browse/MODINVSTOR-1247))
+* Implement publication period migration on big dataset, create new InstanceWithoutPubPeriod schema only for input request ([MODINVSTOR-1271](https://folio-org.atlassian.net/browse/MODINVSTOR-1271))
 
 ### Tech Dept
 * Upgrade localstack from 0.11.3 to s3-latest (=3.8.0) ([MODINVSTOR-1272](https://folio-org.atlassian.net/browse/MODINVSTOR-1272))

diff --git a/descriptors/ModuleDescriptor-template.json b/descriptors/ModuleDescriptor-template.json
@@ -167,7 +167,7 @@
     },
     {
       "id": "instance-storage",
-      "version": "11.0",
+      "version": "11.1",
       "handlers": [
         {
           "methods": ["GET"],
@@ -274,7 +274,7 @@
     },
     {
       "id": "instance-storage-batch",
-      "version": "3.0",
+      "version": "3.1",
       "handlers": [
         {
           "methods": ["POST"],

diff --git a/pom.xml b/pom.xml
@@ -23,6 +23,7 @@
     <commons-lang3.version>3.17.0</commons-lang3.version>
     <log4j.version>2.24.1</log4j.version>
     <folio-s3-client.version>2.2.0-SNAPSHOT</folio-s3-client.version>
+    <commons-beanutils.version>1.9.4</commons-beanutils.version>
 
     <okapi-testing.version>6.0.3</okapi-testing.version>
     <junit.version>5.11.2</junit.version>
@@ -138,6 +139,11 @@
       <artifactId>folio-s3-client</artifactId>
       <version>${folio-s3-client.version}</version>
     </dependency>
+    <dependency>
+      <groupId>commons-beanutils</groupId>
+      <artifactId>commons-beanutils</artifactId>
+      <version>${commons-beanutils.version}</version>
+    </dependency>
 
     <!-- Test Dependencies  -->
     <dependency>

diff --git a/ramls/examples/instance_get.json b/ramls/examples/instance_get.json
@@ -19,6 +19,10 @@
       "value": "1"
     }
   ],
+  "publicationPeriod": {
+    "start": 1999,
+    "end": 2001
+  },
   "instanceTypeId": "2b94c631-fca9-4892-a730-03ee529ffe2c",
   "tags" : {
     "tagList" : [

diff --git a/ramls/examples/instances_get.json b/ramls/examples/instances_get.json
@@ -21,6 +21,10 @@
           "value": "1"
         }
       ],
+      "publicationPeriod": {
+        "start": 1999,
+        "end": 2001
+      },
       "instanceTypeId": "2b94c631-fca9-4892-a730-03ee529ffe2c",
       "tags" : {
         "tagList" : [

diff --git a/ramls/examples/instanceswithoutpubperiod_get.json b/ramls/examples/instanceswithoutpubperiod_get.json
@@ -0,0 +1,56 @@
+{
+  "instances": [
+    {
+      "id": "601a8dc4-dee7-48eb-b03f-d02fdf0debd0",
+      "title": "ADVANCING LIBRARY EDUCATION: TECHNOLOGICAL INNOVATION AND INSTRUCTIONAL DESIGN",
+      "source": "Local: MARC",
+      "contributors": [
+        {
+          "name": "Sigal, Ari",
+          "contributorNameTypeId": "2b94c631-fca9-4892-a730-03ee529ffe2a",
+          "primary": true
+        }
+      ],
+      "identifiers": [
+        {
+          "identifierTypeId": "2e48e713-17f3-4c13-a9f8-23845bb210af",
+          "value": "9781466636897"
+        },
+        {
+          "identifierTypeId": "6051f95c-028e-4c6a-8a9e-ee689dd51453",
+          "value": "1"
+        }
+      ],
+      "instanceTypeId": "2b94c631-fca9-4892-a730-03ee529ffe2c",
+      "tags" : {
+        "tagList" : [
+          "important"
+        ]
+      }
+    },
+    {
+      "id": "f31a36de-fcf8-44f9-87ef-a55d06ad21ae",
+      "title": "ADVANCING RESEARCH METHODS WITH NEW TECHNOLOGIES.",
+      "source": "Local: MARC",
+      "contributors": [
+        {
+          "name": "Sappleton, Natalie",
+          "contributorNameTypeId": "2b94c631-fca9-4892-a730-03ee529ffe2a",
+          "primary": true
+        }
+      ],
+      "identifiers": [
+        {
+          "identifierTypeId": "2e48e713-17f3-4c13-a9f8-23845bb210af",
+          "value": "9781466639195"
+        },
+        {
+          "identifierTypeId": "6051f95c-028e-4c6a-8a9e-ee689dd51453",
+          "value": "2"
+        }
+      ],
+      "instanceTypeId": "2b94c631-fca9-4892-a730-03ee529ffe2c"
+    }
+  ],
+  "totalRecords": 2
+}
diff --git a/ramls/examples/instancewithoutpubperiod_get.json b/ramls/examples/instancewithoutpubperiod_get.json
@@ -0,0 +1,28 @@
+{
+  "id": "601a8dc4-dee7-48eb-b03f-d02fdf0debd0",
+  "source": "Local: MARC",
+  "title": "ADVANCING LIBRARY EDUCATION: TECHNOLOGICAL INNOVATION AND INSTRUCTIONAL DESIGN",
+  "contributors": [
+    {
+      "name": "Sigal, Ari",
+      "contributorNameTypeId": "2b94c631-fca9-4892-a730-03ee529ffe2a",
+      "primary": true
+    }
+  ],
+  "identifiers": [
+    {
+      "identifierTypeId": "2e48e713-17f3-4c13-a9f8-23845bb210af",
+      "value": "9781466636897"
+    },
+    {
+      "identifierTypeId": "6051f95c-028e-4c6a-8a9e-ee689dd51453",
+      "value": "1"
+    }
+  ],
+  "instanceTypeId": "2b94c631-fca9-4892-a730-03ee529ffe2c",
+  "tags" : {
+    "tagList" : [
+      "important"
+    ]
+  }
+}
diff --git a/ramls/instance-storage-batch.raml b/ramls/instance-storage-batch.raml
@@ -11,6 +11,7 @@ documentation:
 types:
   errors: !include raml-util/schemas/errors.schema
   instances: !include instances.json
+  instancesWithoutPubPeriod: !include instances-without-pub-period.json
   instancesBatchResponse: !include instances-batch-response.json
 
 /instance-storage/batch/instances:
@@ -19,7 +20,9 @@ types:
     description: "Create collection of instances in one request - deprecated, use /instance-storage/sync instead"
     body:
       application/json:
-        type: instances
+        type: instancesWithoutPubPeriod
+        example:
+          value: !include examples/instanceswithoutpubperiod_get.json
     responses:
       201:
         description: "At least one Instance from the list was created"

diff --git a/ramls/instance-storage.raml b/ramls/instance-storage.raml
@@ -9,6 +9,8 @@ documentation:
     content: <b>Storage for instances in the inventory</b>
 
 types:
+  instanceWithoutPubPeriod: !include instance-without-pub-period.json
+  instancesWithoutPubPeriod: !include instances-without-pub-period.json
   instance: !include instance.json
   instances: !include instances.json
   marcJson: !include marc.json
@@ -60,16 +62,26 @@ resourceTypes:
     displayName: Instances
     type:
       collection:
-        exampleCollection: !include  examples/instances_get.json
-        schemaCollection: instances
-        schemaItem: instance
-        exampleItem: !include  examples/instance_get.json
+        exampleCollection: !include  examples/instanceswithoutpubperiod_get.json
+        schemaCollection: instancesWithoutPubPeriod
+        schemaItem: instanceWithoutPubPeriod
+        exampleItem: !include  examples/instancewithoutpubperiod_get.json
     get:
       is: [pageable,
           searchable: {description: "by title (using CQL)",
                         example: "title=\"*uproot*\""},
           ]
+      responses:
+        200:
+          body:
+            application/json:
+              type: instances
     post:
+      responses:
+        200:
+          body:
+            application/json:
+              type: instance
     delete:
       is: [searchable: { description: "CQL to select instances to delete, use cql.allRecords=1 to delete all. Deletes connected marc source records.",
                          example: "hrid==\"in123-0*\"" } ]
@@ -89,8 +101,8 @@ resourceTypes:
     /{instanceId}:
       type:
         collection-item:
-          exampleItem: !include examples/instance_get.json
-          schema: instance
+          exampleItem: !include examples/instancewithoutpubperiod_get.json
+          schema: instanceWithoutPubPeriod
       get:
         responses:
           200: