Merge Dev to Staging (#201)

* fix(map view): prevent client error when legend config is empty * Feature/pending datasets (#198) * Add pending_datasets table init and CRUD * Add approval_status and draft to metadata * Fix draft permissions * [main.yml] Init. pendingdb tables * Run WRI unit tests first * Fix paths in unit tests script * Run WRI unit tests first in src * Fix OR in unit tests script --------- Co-authored-by: Muhammad Ismail Shahzad <[email protected]> * Rm test.yml (#202) --------- Co-authored-by: Demenech <[email protected]> Co-authored-by: João Demenech <[email protected]> Co-authored-by: Michael Polidori <[email protected]> Co-authored-by: Muhammad Ismail Shahzad <[email protected]>
wri · Jan 15, 2024 · ef0e84a · ef0e84a
1 parent bc33bf1
commit ef0e84a
Show file tree

Hide file tree

Showing 23 changed files with 897 additions and 175 deletions.
diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
@@ -126,6 +126,8 @@ jobs:
           CKAN_IMAGE: '${{ steps.login-ecr.outputs.registry }}/${{ secrets.ECR_CKAN_REPO }}:${{ github.sha }}'
         run: docker compose -f docker-compose.test.yml --env-file .env.example exec -T ckan-dev /bin/bash -c "/srv/app/fix_s3filestore_test_ini.sh"
         working-directory: ./ckan-backend-dev
+      - name: Initialize the pending datasets table
+        run: docker exec ckan-wri sh -c "ckan -c production.ini pendingdatasetsdb"
       - name: Run Unit Tests 🧪
         env:
           CKAN_IMAGE: '${{ steps.login-ecr.outputs.registry }}/${{ secrets.ECR_CKAN_REPO }}:${{ github.sha }}'

diff --git a/ckan-backend-dev/ckan/docker-entrypoint.d/init_tables.sh b/ckan-backend-dev/ckan/docker-entrypoint.d/init_tables.sh
@@ -7,7 +7,13 @@ ckan -c production.ini notificationdb
 EXIT_CODE=$?
 
 if [ $EXIT_CODE -ne 0 ]; then
-    echo "Failed to initialize custom tables"
+    echo "Failed to initialize the notification table"
+fi
+
+ckan -c production.ini pendingdatasetsdb
+
+if [ $EXIT_CODE -ne 0 ]; then
+    echo "Failed to initialize the pending datasets table"
     exit $EXIT_CODE
 fi
 

diff --git a/ckan-backend-dev/ckan/scripts/init-extensions.sh b/ckan-backend-dev/ckan/scripts/init-extensions.sh
@@ -2,3 +2,4 @@
 # Initialize Issues and Notification DB
 docker exec ckan-wri sh -c "ckan -c production.ini issuesdb"
 docker exec ckan-wri sh -c "ckan -c production.ini notificationdb"
+docker exec ckan-wri sh -c "ckan -c production.ini pendingdatasetsdb"
diff --git a/ckan-backend-dev/ckan/scripts/run_unit_tests.sh b/ckan-backend-dev/ckan/scripts/run_unit_tests.sh
@@ -13,12 +13,40 @@ echo "Test Summary" > "$ROOT_DIR/test_summary.txt"
 #     echo "CKAN Core: Failed" >> test_summary.txt
 # fi
 
-cd src
+if [ -d "src_extensions/ckanext-wri" ]; then
+  cd src_extensions/ckanext-wri
+
+  pytest --ckan-ini=test.ini ckanext/wri/tests 2>&1 | tee -a "$ROOT_DIR/test_results.txt"
+  PYTEST_EXIT_CODE=${PIPESTATUS[0]}
+
+  if [ $PYTEST_EXIT_CODE -eq 0 ]; then
+    echo "ckanext-wri: Passed" >> "$ROOT_DIR/test_summary.txt"
+  else
+    echo "ckanext-wri: Failed" >> "$ROOT_DIR/test_summary.txt"
+  fi
+
+fi
+
+if [ -d "$ROOT_DIR/src/ckanext-wri" ]; then
+  cd $ROOT_DIR/src/ckanext-wri
+
+  pytest --ckan-ini=test.ini ckanext/wri/tests 2>&1 | tee -a "$ROOT_DIR/test_results.txt"
+  PYTEST_EXIT_CODE=${PIPESTATUS[0]}
+
+  if [ $PYTEST_EXIT_CODE -eq 0 ]; then
+    echo "ckanext-wri: Passed" >> "$ROOT_DIR/test_summary.txt"
+  else
+    echo "ckanext-wri: Failed" >> "$ROOT_DIR/test_summary.txt"
+  fi
+
+fi
+
+cd $ROOT_DIR/src
 
 for dir in ckanext-*; do
   if [ -d "$dir" ]; then
     # Skip ckanext-envvars
-    if [ "$dir" == "ckanext-envvars" ]; then
+    if [ "$dir" == "ckanext-envvars" ] || [ "$dir" == "ckanext-wri" ]; then
       continue
     fi
 
@@ -38,24 +66,6 @@ for dir in ckanext-*; do
   fi
 done
 
-cd ..
-
-if [ -d "src_extensions/ckanext-wri" ]; then
-  cd src_extensions/ckanext-wri
-
-  pytest --ckan-ini=test.ini ckanext/wri/tests 2>&1 | tee -a "$ROOT_DIR/test_results.txt"
-  PYTEST_EXIT_CODE=${PIPESTATUS[0]}
-
-  if [ $PYTEST_EXIT_CODE -eq 0 ]; then
-    echo "ckanext-wri: Passed" >> "$ROOT_DIR/test_summary.txt"
-  else
-    echo "ckanext-wri: Failed" >> "$ROOT_DIR/test_summary.txt"
-  fi
-
-  cd ..
-
-fi
-
 cat "$ROOT_DIR/test_summary.txt"
 
 # GitHub Actions failure exit code

diff --git a/ckan-backend-dev/ckan/setup/schema.xml b/ckan-backend-dev/ckan/setup/schema.xml
@@ -203,6 +203,8 @@ attribute with the form `ckan-X.Y` -->
     <field name="wri_data" type="boolean" indexed="true" stored="true"/>
     <field name="spatial_geom" type="location_rpt" indexed="true" multiValued="true" />
     <field name="spatial_address" type="string" indexed="true" stored="true" multiValued="true"/>
+    <field name="approval_status" type="string" indexed="true" stored="true"/>
+    <field name="draft" type="boolean" indexed="true" stored="true"/>
 </fields>
 
 <uniqueKey>index_id</uniqueKey>

diff --git a/ckan-backend-dev/src/ckanext-wri/README.md b/ckan-backend-dev/src/ckanext-wri/README.md
@@ -3,6 +3,21 @@
 **Table of Contents**
 
 - [ckanext-wri](#ckanext-wri)
+  - [Notifications Feature](#notifications-feature)
+    - [Database Setup](#database-setup)
+    - [API Endpoints](#api-endpoints)
+      - [POST /api/action/notification_create](#post-apiactionnotification_create)
+      - [POST /api/action/notification_update](#post-apiactionnotification_update)
+      - [GET /api/action/notification_get_all](#get-apiactionnotification_get_all)
+  - [Pending Datasets (Approval Workflow)](#pending-datasets-approval-workflow)
+    - [Pending Dataset Table](#pending-dataset-table)
+      - [Initializing the Pending Dataset Table](#initializing-the-pending-dataset-table)
+    - [API Endpoints](#api-endpoints-1)
+      - [POST /api/action/pending_dataset_create](#post-apiactionpending_dataset_create)
+      - [POST /api/action/pending_dataset_update](#post-apiactionpending_dataset_update)
+      - [POST /api/action/pending_dataset_delete](#post-apiactionpending_dataset_delete)
+      - [GET /api/action/pending_dataset_show](#get-apiactionpending_dataset_show)
+      - [GET /api/action/pending_diff_show](#get-apiactionpending_diff_show)
   - [Development](#development)
   - [Testing](#testing)
 
@@ -62,6 +77,117 @@ Returns a list of notifications for a sender or recipient.
 - **recipient_id** (string) – The user ID of the recipient of the notification (optional, but either `recipient_id` or `sender_id` is required).
 - **sender_id** (string) – The user ID of the sender of the notification (optional, but either `recipient_id` or `sender_id` is required).
 
+## Pending Datasets (Approval Workflow)
+
+A pending dataset is dataset metadata that's been submitted for approval. While pending, the dataset metadata lives in a separate table from the main `package` table, `pending_datasets`. Once approved, the existing dataset is updated with the new metadata.
+
+### Pending Dataset Table
+
+The `pending_datasets` table has the following columns:
+
+| `package_id` | `package_data` | `last_modified` |
+| ------------ | -------------- | --------------- |
+| `text` (PK)  | `jsonb`        | `timestamp`     |
+
+The `package_id` column is the UUID of the dataset (and it's the primary key). The `package_data` column contains the dataset metadata as a JSONB object. The `last_modified` column is a timestamp that is automatically generated whenever `package_data` is updated.
+
+#### Initializing the Pending Dataset Table
+
+You can initialize the pending dataset table by running the following command:
+
+```console
+ckan -c <path-to-ini-file> pendingdatasetsdb
+```
+
+### API Endpoints
+
+#### POST /api/action/pending_dataset_create
+
+**Parameters:**
+- **package_id** (string) – The UUID of the dataset (required).
+- **package_data** (JSON object) – The dataset metadata (required).
+
+Creates a new pending dataset and returns the newly created pending dataset.
+
+#### POST /api/action/pending_dataset_update
+
+**Parameters:**
+- **package_id** (string) – The UUID of the dataset (required).
+- **package_data** (JSON object) – The dataset metadata (required).
+
+Updates an existing pending dataset and returns the updated pending dataset.
+
+#### POST /api/action/pending_dataset_delete
+
+**Parameters:**
+- **package_id** (string) – The UUID of the dataset (required).
+
+Deletes an existing pending dataset.
+
+#### GET /api/action/pending_dataset_show
+
+**Parameters:**
+- **package_id** (string) – The UUID of the dataset (required).
+
+Returns the pending dataset for the given `package_id`.
+
+#### GET /api/action/pending_diff_show
+
+**Parameters:**
+- **package_id** (string) – The UUID of the dataset (required).
+
+Returns the diff between the pending dataset and the existing dataset for the given `package_id`.
+
+Here's an example:
+
+```json
+{
+    "help": "http://ckan-dev:5000/api/3/action/help_show?name=pending_diff_show",
+    "success": true,
+    "result": {
+        "title": {
+            "old_value": "My dataset title",
+            "new_value": "My better dataset title"
+        },
+        "application": {
+            "old_value": "",
+            "new_value": "wri"
+        },
+        "resources[0].description": {
+            "old_value": "My resource description",
+            "new_value": "My better resource description"
+        },
+        "resources[0].format": {
+            "old_value": "CSV",
+            "new_value": "HTML"
+        },
+        "resources[1].title": {
+            "old_value": "My resource title",
+            "new_value": "My better resource title"        },
+        "wri_data": {
+            "old_value": false,
+            "new_value": true
+        },
+        "cautions": {
+            "old_value": "",
+            "new_value": "This is a caution"
+        },
+        "languages": {
+            "old_value": [
+                "fr"
+            ],
+            "new_value": [
+                "en"
+            ]
+        },
+        "function": {
+            "old_value": "The function of this dataset is to x...",
+            "new_value": "The function of this dataset is to y..."
+        },
+    },
+}
+```
+
 ## Development
 
 See the [CKAN Backend Development README](ckan-backend-dev/README.md) for instructions on how to set up a local Docker CKAN backend development environment.

diff --git a/ckan-backend-dev/src/ckanext-wri/ckanext/wri/logic/action/create.py b/ckan-backend-dev/src/ckanext-wri/ckanext/wri/logic/action/create.py
@@ -1,18 +1,25 @@
-from ckan.types import ActionResult, Context, DataDict
 from typing_extensions import TypeAlias
 import logging
+
 from ckanext.wri.model.notification import Notification, notification_dictize
-import ckan.plugins.toolkit as tk
+from ckanext.wri.model.pending_datasets import PendingDatasets
 from ckanext.wri.logic.auth import schema
 
+from ckan.common import _
+import ckan.plugins.toolkit as tk
+from ckan.types import Context, DataDict
+
 NotificationGetUserViewedActivity: TypeAlias = None
 log = logging.getLogger(__name__)
 
-def notification_create(context: Context, data_dict: DataDict) -> NotificationGetUserViewedActivity:
+
+def notification_create(
+    context: Context, data_dict: DataDict
+) -> NotificationGetUserViewedActivity:
     """Create a Notification by providing Sender and Recipient"""
-    
+
     model = context["model"]
-    session = context['session']
+    session = context["session"]
     user_obj = model.User.get(context["user"])
 
     tk.check_access("notification_create", context, data_dict)
@@ -21,24 +28,52 @@ def notification_create(context: Context, data_dict: DataDict) -> NotificationGe
     if errors:
         raise tk.ValidationError(errors)
 
-    recipient_id = data_dict.get('recipient_id')
-    sender_id = data_dict.get('sender_id')
-    activity_type = data_dict.get('activity_type')
-    object_type = data_dict.get('object_type')
-    object_id = data_dict.get('object_id')
+    recipient_id = data_dict.get("recipient_id")
+    sender_id = data_dict.get("sender_id")
+    activity_type = data_dict.get("activity_type")
+    object_type = data_dict.get("object_type")
+    object_id = data_dict.get("object_id")
 
     user_notifications = Notification(
         recipient_id=recipient_id,
         sender_id=sender_id,
         activity_type=activity_type,
         object_type=object_type,
-        object_id=object_id
+        object_id=object_id,
     )
 
     session.add(user_notifications)
 
-    if not context.get('defer_commit'):
+    if not context.get("defer_commit"):
         model.repo.commit()
 
     notification_dicts = notification_dictize(user_notifications, context)
     return notification_dicts
+
+
+def pending_dataset_create(context: Context, data_dict: DataDict):
+    """Create a Pending Dataset"""
+    package_id = data_dict.get("package_id")
+    package_data = data_dict.get("package_data")
+    log.error(package_data)
+
+    if not package_id:
+        raise tk.ValidationError(_("package_id is required"))
+
+    if not package_data:
+        raise tk.ValidationError(_("package_data is required"))
+
+    tk.check_access("pending_dataset_create", context, package_data)
+
+    pending_dataset = None
+
+    try:
+        pending_dataset = PendingDatasets.create(package_id, package_data)
+    except Exception as e:
+        log.error(e)
+        raise tk.ValidationError(e)
+
+    if not pending_dataset:
+        raise tk.ValidationError(_(f"Pending Dataset not found: {package_id}"))
+
+    return pending_dataset
diff --git a/ckan-backend-dev/src/ckanext-wri/ckanext/wri/logic/action/delete.py b/ckan-backend-dev/src/ckanext-wri/ckanext/wri/logic/action/delete.py
@@ -0,0 +1,32 @@
+import logging
+
+from ckan.types import Context, DataDict
+import ckan.plugins.toolkit as tk
+from ckan.common import _
+
+from ckanext.wri.model.pending_datasets import PendingDatasets
+
+log = logging.getLogger(__name__)
+
+
+def pending_dataset_delete(context: Context, data_dict: DataDict):
+    """Delete a Pending Dataset"""
+    package_id = data_dict.get("package_id")
+
+    if not package_id:
+        raise tk.ValidationError(_("package_id is required"))
+
+    tk.check_access("pending_dataset_delete", context, data_dict)
+
+    pending_dataset = None
+
+    try:
+        pending_dataset = PendingDatasets.delete(package_id)
+    except Exception as e:
+        log.error(e)
+        raise tk.ValidationError(e)
+
+    if not pending_dataset:
+        raise tk.ValidationError(_(f"Pending Dataset not found: {package_id}"))
+
+    return pending_dataset