Merge pull request #523 from NASA-IMPACT/bugfix/casei-538-jsonfields

Restore JSONField columns on DOI model.

.dockerignore

@@ -0,0 +1,13 @@
+__pycache__
+.github
+.pytest_cache
+env
+out
+staticfiles
+.dockerignore
+.env
+.env.*
+*.json
+docker-compose*
+Dockerfile*
+.backup

.gitignore

@@ -290,3 +290,5 @@ cmr/config.py
 *.sql
 hosts/*
 !hosts/*.sample
+
+.backup

Dockerfile.local

@@ -1,5 +1,5 @@
 # pull official base image
-FROM python:3.11.1 AS builder
+FROM --platform=linux/amd64 python:3.11.1 AS builder
 
 # set environment variables
 ENV PYTHONDONTWRITEBYTECODE 1
@@ -9,8 +9,7 @@ ENV PYTHONUNBUFFERED 1
 WORKDIR /home/app/web
 
 # install dependencies
-RUN apt-get update --fix-missing
-RUN apt-get install -y gdal-bin libgdal-dev postgis netcat
+RUN apt-get update --fix-missing && apt-get install -y gdal-bin libgdal-dev postgis netcat
 
 # install requirements
 COPY requirements/ .

Dockerfile.prod

@@ -1,5 +1,5 @@
 # pull official base image
-FROM python:3.11.1 AS builder
+FROM --platform=linux/amd64 python:3.11.1 AS builder
 
 # set environment variables
 ENV PYTHONDONTWRITEBYTECODE 1
@@ -9,8 +9,7 @@ ENV PYTHONUNBUFFERED 1
 WORKDIR /home/app/web
 
 # install dependencies
-RUN apt-get update --fix-missing
-RUN apt-get install -y gdal-bin libgdal-dev postgis netcat
+RUN apt-get update --fix-missing && apt-get install -y gdal-bin libgdal-dev postgis netcat
 
 # install requirements
 COPY requirements/ .

admin_ui/mixins.py

@@ -57,7 +57,15 @@ def formfield_callback(f, disabled_fields=[], **kwargs):
     elif isinstance(f, model_fields.BooleanField):
         # Adding choices assigns a "yes/no" option and creates a dropdown widget
         f.choices = ((True, "Yes"), (False, "No"))
-
+    elif isinstance(f, model_fields.ManyToManyField):
+        if f.remote_field.model == models.CollectionPeriod:
+            kwargs.update(
+                {
+                    "queryset": models.CollectionPeriod.objects.select_related(
+                        "deployment", "deployment__campaign", "platform"
+                    )
+                }
+            )
     return f.formfield(disabled=f.name in disabled_fields, **kwargs)
 
 
@@ -135,12 +143,21 @@ def get_help_texts(model_type) -> Dict[str, str]:
         if model_type == models.Campaign:
             return {
                 "short_name": "Abbreviation for field investigation name (typically an acronym)",
-                "long_name": "Full name of field investigation (typically the acronym fully spelled out)",
+                "long_name": (
+                    "Full name of field investigation (typically the acronym fully spelled out)"
+                ),
             }
         elif model_type == models.Deployment:
             return {
-                "short_name": "Format as “dep_YYYY[i]” with YYYY as the year in which deployment begins, and optional lowercase character (i=a, b, …) appended only if there are multiple deployments in a single calendar year for the campaign",
-                "long_name": "If there are named sub-campaigns, the name used for this deployment (e.g. CAMEX-3).  This may not exist.",
+                "short_name": (
+                    "Format as “dep_YYYY[i]” with YYYY as the year in which deployment begins, and"
+                    " optional lowercase character (i=a, b, …) appended only if there are multiple"
+                    " deployments in a single calendar year for the campaign"
+                ),
+                "long_name": (
+                    "If there are named sub-campaigns, the name used for this deployment (e.g."
+                    " CAMEX-3).  This may not exist."
+                ),
             }
         elif model_type == models.Platform:
             return {
@@ -154,7 +171,10 @@ def get_help_texts(model_type) -> Dict[str, str]:
             }
         elif model_type == models.SignificantEvent:
             return {
-                "short_name": "ADMG's text identifier for the SE - format as 'XXX_SE_#' with XXX as the campaign shortname and # as the integer number of the SE within the campaign"
+                "short_name": (
+                    "ADMG's text identifier for the SE - format as 'XXX_SE_#' with XXX as the"
+                    " campaign shortname and # as the integer number of the SE within the campaign"
+                )
             }
         else:
             return {}

admin_ui/tables/published.py

@@ -57,6 +57,9 @@ class IOPPublishedTable(tables.Table):
     start_date = tables.Column(verbose_name="Start Date", accessor="start_date")
     end_date = tables.Column(verbose_name="End Date", accessor="end_date")
 
+    def __init__(self, data, *args, **kwargs):
+        super().__init__(data.select_related("deployment"), *args, **kwargs)
+
     class Meta(LimitedTableBase.Meta):
         fields = ["short_name", "deployment", "start_date", "end_date"]
         sequence = fields
@@ -78,6 +81,9 @@ class SignificantEventPublishedTable(tables.Table):
     start_date = tables.Column(verbose_name="Start Date", accessor="start_date")
     end_date = tables.Column(verbose_name="End Date", accessor="end_date")
 
+    def __init__(self, data, *args, **kwargs):
+        super().__init__(data.select_related("deployment"), *args, **kwargs)
+
     class Meta(LimitedTableBase.Meta):
         model = SignificantEvent
         fields = ("short_name", "deployment", "start_date", "end_date")
@@ -100,6 +106,13 @@ class CollectionPeriodPublishedTable(tables.Table):
         verbose_name="Instruments", model=Instrument, accessor="instruments"
     )
 
+    def __init__(self, data, *args, **kwargs):
+        super().__init__(
+            data.select_related("deployment", "platform").prefetch_related("instruments"),
+            *args,
+            **kwargs
+        )
+
     class Meta(LimitedTableBase.Meta):
         model = CollectionPeriod
         fields = ("deployment", "platform", "instruments")
@@ -123,6 +136,11 @@ class DOIPublishedTable(tables.Table):
         verbose_name="Instruments", model=Instrument, accessor="instruments"
     )
 
+    def __init__(self, data, *args, **kwargs):
+        super().__init__(
+            data.prefetch_related("campaigns", "platforms", "instruments"), *args, **kwargs
+        )
+
     class Meta(LimitedTableBase.Meta):
         model = DOI
         fields = ("concept_id", "long_name", "campaigns", "platforms", "instruments")
@@ -139,6 +157,9 @@ class DeploymentPublishedTable(LimitedTableBase):
     start_date = tables.Column(verbose_name="Start Date", accessor="start_date")
     end_date = tables.Column(verbose_name="End Date", accessor="end_date")
 
+    def __init__(self, data, *args, **kwargs):
+        super().__init__(data.select_related("campaign"), *args, **kwargs)
+
     class Meta(LimitedTableBase.Meta):
         fields = LimitedTableBase.initial_fields + ("campaign", "start_date", "end_date")
         sequence = fields

admin_ui/tables/tables.py

@@ -111,10 +111,37 @@ def get_short_name(self, potential_uuid):
             # this really should never happen
             return potential_uuid
 
+    def get_short_names(self, potential_uuids):
+        short_names = {}
+        lookup_uuids = []
+        for potential_uuid in potential_uuids:
+            if not self.is_uuid(potential_uuid):
+                short_names[str(potential_uuid)] = potential_uuid
+                continue
+            else:
+                lookup_uuids.append(str(potential_uuid))
+        if lookup_uuids:
+            model_objects = self.model.objects.filter(uuid__in=lookup_uuids)
+            short_names.update(
+                {str(model_object.uuid): model_object.short_name for model_object in model_objects}
+            )
+            missing_uuids = set(lookup_uuids) - set(short_names.keys())
+            if missing_uuids:
+                change_objects = Change.objects.filter(uuid__in=missing_uuids)
+                short_names.update(
+                    {
+                        str(change_object.uuid): change_object.update.get(
+                            "short_name", change_object.uuid
+                        )
+                        for change_object in change_objects
+                    }
+                )
+        return [short_names[str(potential_uuid)] for potential_uuid in potential_uuids]
+
     def render(self, **kwargs):
         value = self.get_backup_value(**kwargs)
         if isinstance(value, list):
-            return ", ".join(self.get_short_name(potential_uuid) for potential_uuid in value)
+            return ", ".join(self.get_short_names(value))
         else:
             return self.get_short_name(value)
 
@@ -479,8 +506,10 @@ class Meta(LimitedTableBase.Meta):
 
     def render_short_name(self, value, record):
         return format_html(
-            '<a href="{form_url}">{label}</a> <a href="{dashboard_url}" class="font-italic'
-            ' small">(dashboard)</a>',
+            (
+                '<a href="{form_url}">{label}</a> <a href="{dashboard_url}" class="font-italic'
+                ' small">(dashboard)</a>'
+            ),
             form_url=reverse('change-update', args=[record.uuid]),
             label=record.update.get('short_name') or '---',
             dashboard_url=reverse('campaign-detail', args=[record.uuid]),

admin_ui/tests/test_views/test_published.py

@@ -0,0 +1,64 @@
+import pytest
+from django.urls import reverse
+
+from admin_ui.tables.tables import ShortNamefromUUIDColumn
+from admin_ui.tests.factories import ChangeFactory
+from data_models.tests.factories import CampaignFactory
+from data_models.models import Campaign
+
+
+@pytest.mark.django_db
+@pytest.mark.parametrize(
+    "model_name",
+    [
+        "campaign",
+        "doi",
+        "collection_period",
+    ],
+)
+class TestPublished:
+    def test_get(self, client, model_name):
+        url = reverse("published-list", args=(model_name,))
+        response = client.get(url)
+        assert 200 == response.status_code
+
+
+@pytest.mark.django_db
+class TestShortNameFromUUIDColumn:
+    def test_get_names_from_concrete_model(self):
+        campaign = CampaignFactory()
+        column = ShortNamefromUUIDColumn(Campaign)
+        assert column.get_short_names([str(campaign.uuid)])[0] == campaign.short_name
+
+    def test_get_name_from_concrete_model(self):
+        campaign = CampaignFactory()
+        column = ShortNamefromUUIDColumn(Campaign)
+        assert column.get_short_name(str(campaign.uuid)) == campaign.short_name
+
+    def test_get_names_from_change_model(self):
+        change = ChangeFactory.make_create_change_object(
+            CampaignFactory, custom_fields={"short_name": "test"}
+        )
+        column = ShortNamefromUUIDColumn(Campaign)
+        assert column.get_short_names([str(change.uuid)])[0] == change.update["short_name"]
+
+    def test_get_name_from_change_model(self):
+        change = ChangeFactory.make_create_change_object(
+            CampaignFactory, custom_fields={"short_name": "test"}
+        )
+        column = ShortNamefromUUIDColumn(Campaign)
+        assert column.get_short_name(str(change.uuid)) == change.update["short_name"]
+
+    def test_get_names_from_non_uuid_values(self):
+        """
+        Should return the provided values intact
+        """
+        column = ShortNamefromUUIDColumn(Campaign)
+        assert column.get_short_names(["test"]) == ["test"]
+
+    def test_get_name_from_non_uuid_value(self):
+        """
+        Should return the provided value intact
+        """
+        column = ShortNamefromUUIDColumn(Campaign)
+        assert column.get_short_name("test") == "test"

admin_ui/utils.py

@@ -1,4 +1,4 @@
-from django.forms import FileField, ModelForm
+from django.forms import FileField, JSONField, ModelForm
 
 
 def compare_values(old_item, new_item):
@@ -46,7 +46,8 @@ def serialize_model_form(model_form: ModelForm):
                 continue
             model_field.save(model_field.url, model_form.cleaned_data[name])
             update[name] = model_field.name
-
+        elif isinstance(field, JSONField):
+            update[name] = model_form.cleaned_data[name]
         else:
             # Populate Change's form with values from destination model's form.
             # We're not saving the cleaned_data because we want the raw text, not

config/settings/local.py

@@ -46,7 +46,7 @@
     MIDDLEWARE += ["debug_toolbar.middleware.DebugToolbarMiddleware"]  # noqa F405
     # https://django-debug-toolbar.readthedocs.io/en/latest/configuration.html#debug-toolbar-config
     DEBUG_TOOLBAR_CONFIG = {
-        "DISABLE_PANELS": ["debug_toolbar.panels.redirects.RedirectsPanel"],
+        # "DISABLE_PANELS": ["debug_toolbar.panels.redirects.RedirectsPanel"],
         "SHOW_TEMPLATE_CONTEXT": True,
     }
 # https://django-debug-toolbar.readthedocs.io/en/latest/installation.html#internal-ips

data_models/migrations/0055_auto_20230613_1348.py

@@ -0,0 +1,49 @@
+# Generated by Django 4.1.5 on 2023-06-13 18:48
+
+import ast
+import json
+import logging
+from django.db import migrations
+
+logger = logging.getLogger(__name__)
+
+fields_to_convert = [
+    "cmr_projects",
+    "cmr_dates",
+    "cmr_plats_and_insts",
+    "cmr_science_keywords",
+    "cmr_data_formats",
+]
+
+
+def convert_cmr_data_to_json(apps, schema_editor):
+    DOI = apps.get_model("data_models", "DOI")
+
+    dois = DOI.objects.all()
+    for doi in dois:
+        for field in fields_to_convert:
+            logger.info(f"Converting {field} on {doi.uuid} to JSON")
+            d = getattr(doi, field)
+            if d is None or d == "":
+                logger.info(f"Field {field} is empty, setting to null")
+                json_data = "null"
+            else:
+                try:
+                    json_data = json.loads(d)
+                except json.decoder.JSONDecodeError:
+                    logger.info(
+                        f"Failed to parse JSON, attempting to convert to JSON from serialized python object {d}"
+                    )
+                    json_data = ast.literal_eval(d)
+            setattr(doi, field, json.dumps(json_data))
+        doi.save()
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ('data_models', '0054_remove_instrument_additional_metadata'),
+    ]
+
+    operations = [
+        migrations.RunPython(convert_cmr_data_to_json, reverse_code=migrations.RunPython.noop)
+    ]

data_models/migrations/0056_alter_doi_cmr_data_formats_alter_doi_cmr_dates_and_more.py

@@ -0,0 +1,37 @@
+# Generated by Django 4.1.5 on 2023-06-13 19:23
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ('data_models', '0055_auto_20230613_1348'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='doi',
+            name='cmr_data_formats',
+            field=models.JSONField(blank=True, default=None, null=True),
+        ),
+        migrations.AlterField(
+            model_name='doi',
+            name='cmr_dates',
+            field=models.JSONField(blank=True, default=None, null=True),
+        ),
+        migrations.AlterField(
+            model_name='doi',
+            name='cmr_plats_and_insts',
+            field=models.JSONField(blank=True, default=None, null=True),
+        ),
+        migrations.AlterField(
+            model_name='doi',
+            name='cmr_projects',
+            field=models.JSONField(blank=True, default=None, null=True),
+        ),
+        migrations.AlterField(
+            model_name='doi',
+            name='cmr_science_keywords',
+            field=models.JSONField(blank=True, default=None, null=True),
+        ),
+    ]