Integrate with databus.openenergyplatform.org as backend service

.python-version

@@ -0,0 +1 @@
+3.10

databus/admin.py

@@ -0,0 +1,3 @@
+from django.contrib import admin  # noqa:F401
+
+# Register your models here.

databus/apps.py

@@ -0,0 +1,6 @@
+from django.apps import AppConfig
+
+
+class DatabusConfig(AppConfig):
+    default_auto_field = "django.db.models.BigAutoField"
+    name = "databus"

databus/databus.py

@@ -0,0 +1,169 @@
+import logging
+from typing import Optional
+
+import databusclient
+import requests
+
+from databus.errors import DeployError, MossError
+from dataedit.metadata import load_metadata_from_db
+from oeplatform.settings import (  # noqa
+    DATABUS_URI_BASE,
+    ENERGY_DATABUS_API_KEY,
+    ENERGY_DATABUS_TEST_GROUP,
+    ENERGY_DATABUS_USERNAME,
+    MOSS_URL,
+    URL,
+)
+
+logger = logging.getLogger("oeplatform")
+
+
+def register_oep_table(
+    schema_name: str,
+    table_name: str,
+    group: str = ENERGY_DATABUS_TEST_GROUP,
+    api_key: str = ENERGY_DATABUS_API_KEY,
+    account_name: str = ENERGY_DATABUS_USERNAME,
+    version: Optional[str] = None,
+    artifact_name: Optional[str] = None,
+    version_column: Optional[str] = None,
+):
+    """
+    @Credit to https://github.com/open-modex/oedatamodel_api/blob/main/oedatamodel_api/databus.py  # noqa: E501
+    Registers OEP table on DataBus and MOSS
+
+    Parameters
+    ----------
+    schema_name: str
+        OEP schema where table is found
+    table_name: str
+        OEP table to register on databus
+    group: str
+        Databus group to deploy to
+    account_name: str
+        Databus account name
+    api_key: str
+        Databus API key
+    version: str
+        defines for which version table is filtered and registered
+    artifact_name: Optional[str]
+        set artifact name, if not set table_name is used
+    version_column: str
+        defines which column shall represent version
+
+    Returns
+    -------
+    databus_identifier: str
+        Databus ID
+    """
+    logger.info(
+        f"Registering table '{schema_name}.{table_name}'"
+        f"in group '{account_name}/{group}' "
+        f"with {version=}"
+    )
+    metadata = load_metadata_from_db(schema_name, table_name)
+    abstract = metadata["description"]
+    license_ = metadata["licenses"][0]["path"]
+
+    url_schema = f"http://{URL}:8000"
+
+    if version_column:
+        if not version:
+            raise Exception(
+                "The version name is missing. Please provide the name of the"
+                "version that is available in the version column."
+            )
+
+        data_url = (
+            f"{url_schema}/api/v0/schema/{schema_name}/tables/{table_name}/rows?"
+            f"form=csv&where={version_column}={version}"
+        )
+    else:
+        if not version:  # TODO Is version required?
+            raise Exception(
+                "The version name is missing."
+                "Please provide a name for the version you want to use."
+            )
+        data_url = (
+            f"{url_schema}/api/v0/schema/{schema_name}/tables/{table_name}/rows?"
+            f"form=csv"
+        )
+
+    metadata_url = f"{url_schema}/api/v0/schema/{schema_name}/tables/{table_name}/meta/"
+
+    distributions = [
+        databusclient.create_distribution(
+            url=data_url,
+            cvs={"type": "data"},
+            file_format="csv",
+        ),
+        databusclient.create_distribution(
+            url=metadata_url,
+            cvs={"type": "metadata"},
+            file_format="json",
+        ),
+    ]
+    artifact_name = artifact_name if artifact_name else table_name
+    version_id = get_databus_identifier(account_name, group, artifact_name, version)
+    dataset = databusclient.create_dataset(
+        version_id,
+        title=metadata["title"],
+        abstract=abstract,
+        description=metadata.get("description", ""),
+        license_url=license_,
+        distributions=distributions,
+    )
+
+    # this function cant send a request!!
+    try:
+        databusclient.deploy(dataset, api_key)
+    except databusclient.client.DeployError as de:
+        raise DeployError(str(de))
+
+    return version_id
+
+
+def submit_metadata_to_moss(databus_identifier, metadata):
+    """
+    Submits metadata from DataBus artifact to MOSS
+
+    Parameters
+    ----------
+    databus_identifier: str
+        Databus ID to set up metadata on MOSS
+    metadata: dict
+        Metadata which shall be connected with databus ID
+
+    Raises
+    ------
+    MossError
+        if metadata cannot be submitted to MOSS
+    """
+    # generate the URI for the request with the encoded identifier
+    # why was there a ?id=({quote
+    api_uri = f"{MOSS_URL}?id={(databus_identifier)}"
+    response = requests.put(
+        api_uri, headers={"Content-Type": "application/ld+json"}, json=metadata
+    )
+    if response.status_code != 200:
+        raise MossError(
+            f"Could not submit metadata for DI '{databus_identifier}' to MOSS. "
+            f"Reason: {response.text}"
+        )
+
+
+def get_databus_identifier(
+    account_name: str, group: str, artifact_name: str, version: Optional[str] = None
+):
+    identifier = f"{DATABUS_URI_BASE}/{account_name}/{group}/{artifact_name}"
+    if version:
+        identifier += f"/{version}"
+    return identifier
+
+
+# Move to frontend
+def check_if_artifact_exists(identifier: str):
+    response = requests.get(identifier)
+    if response.status_code == 200:
+        return True
+    return False

databus/errors.py

@@ -0,0 +1,10 @@
+class MetadataError(Exception):
+    """Raised if metadata is invalid"""
+
+
+class DeployError(Exception):
+    """Raised if deploy fails"""
+
+
+class MossError(Exception):
+    """Raised if submitting metadata to MOSS fails"""

databus/models.py

@@ -0,0 +1,3 @@
+from django.db import models  # noqa
+
+# Create your models here.

databus/templates/databus/partials/register-success.html

@@ -0,0 +1 @@
+<div>Yehaa</div>

databus/templates/databus/partials/register.html

@@ -0,0 +1,35 @@
+{% load django_bootstrap5 %}
+
+<div class="htmxComponent">
+    <button class="btn"
+    hx-post="{% url 'databus:register' schema table %}" 
+    hx-trigger="click"
+    hx-swap="innerHTML">
+        Register
+    </button>
+    <div id="htmx-databus-response-error">
+
+    </div>
+</div>
+
+<script>
+    document.body.addEventListener('htmx:responseError', function(event) {
+        // Identify the form that triggered the error
+        var button = event.target;
+
+        // Use the button's data attribute to find where to display the error message
+        // var errorDisplayId = button.getAttribute('error-display');
+        var errorMessageDiv = document.getElementById('htmx-databus-response-error');
+
+        // Parse the error message from the response
+        var response = JSON.parse(event.detail.xhr.responseText);
+
+        // Handle different error keys appropriately
+        if (response.err_leave) { // For the alter user permissions error
+            errorMessageDiv.innerText = response.err_leave;
+        }
+
+        errorMessageDiv.style.display = 'block'; // Make the error message visible
+        errorMessageDiv.style.color = 'red'; // Optional styling
+    });
+</script>

databus/tests.py

@@ -0,0 +1,3 @@
+from django.test import TestCase  # noqa
+
+# Create your tests here.

databus/urls.py

@@ -0,0 +1,13 @@
+from django.urls import path, re_path  # noqa:F401
+
+from databus import views  # noqa:F401
+
+app_name = "databus"
+
+urlpatterns = [
+    path(
+        "distribution/table/<str:schema>/<str:table>/register/",
+        views.DatabusRegister.as_view(),
+        name="register",
+    )
+]

databus/views.py

@@ -0,0 +1,31 @@
+from django.contrib.auth.mixins import LoginRequiredMixin
+from django.http import HttpResponse, JsonResponse
+from django.shortcuts import render
+from django.views import View  # noqa:F401
+
+from databus.databus import register_oep_table
+
+# Create your views here.
+
+
+class DatabusRegister(View, LoginRequiredMixin):
+    def get(self, request, schema, table):
+        return render(
+            request,
+            "databus/partials/register.html",
+            {"schema": schema, "table": table},
+        )
+
+    def post(self, request, schema, table):
+        if not schema or not table:
+            return JsonResponse({"error": "Invalid schema or table"}, status=400)
+
+        try:
+            register_oep_table(schema_name=schema, table_name=table, version="initial")
+            return render(
+                request,
+                "databus/partials/register-success.html",
+                {"success": f"Table {table} registered in schema {schema}"},
+            )
+        except Exception as e:
+            return HttpResponse({"error": str(e)}, status=500)

dataedit/metadata/__init__.py

@@ -1,3 +1,5 @@
+from metadata.v160.template import OEMETADATA_V160_TEMPLATE
+
 from dataedit.metadata import v1_4 as __LATEST
 from dataedit.metadata.v1_3 import TEMPLATE_v1_3
 from dataedit.metadata.v1_4 import TEMPLATE_V1_4
@@ -6,6 +8,7 @@
 from .error import MetadataException
 
 METADATA_TEMPLATE = {
+    6: OEMETADATA_V160_TEMPLATE,
     5: TEMPLATE_V1_5,
     4: TEMPLATE_V1_4,
     3: TEMPLATE_v1_3,
@@ -173,7 +176,7 @@ def parse_meta_data(metadata, schema, table):
     # if "error" in metadata:
     #     return metadata
     if not metadata:
-        metadata = __LATEST.get_empty(schema, table)
+        metadata = OEMETADATA_V160_TEMPLATE
     else:
         if "error" in metadata:
             return {"description": metadata["content"], "error": metadata["error"]}
@@ -219,37 +222,6 @@ def parse_meta_data(metadata, schema, table):
     return metadata
 
 
-def read_metadata_from_post(content_query, schema, table):
-    """Prepare dict to modify the comment prop of a table in OEP database
-    i.e. contains the metadata
-
-    :param content_query: the content of the POST request
-
-    :param schema: name of the OEP schema
-    :param table: name of the OEP table in the OEP schema
-    :return: metadata dict
-    """
-    version = get_metadata_version(content_query)
-    if version is tuple and len(version) > 2:
-        template = METADATA_TEMPLATE[version[1]].copy()
-    else:
-        template = METADATA_TEMPLATE[4].copy()
-    metadata = assign_content_values_to_metadata(
-        content=content_query, template=template
-    )
-    # TODO fill the "resource" field for v1.4
-    # d["resources"] = [
-    #     {
-    #         "name": "%s.%s" % (schema, table),
-    #         "format": "PostgreSQL",
-    #         "fields": d["field"],
-    #     }
-    # ]
-    # d["metadata_version"] = "1.3"
-
-    return metadata
-
-
 def get_metadata_version(metadata):
     """Find the metadata version in the metadata
 

dataedit/static/metaedit/metaedit.js

@@ -137,11 +137,18 @@ var MetaEdit = function(config) {
     // these will be removed at the end
     function fixRecursive(schemaProps, elemObject, path) {
       // is object ?
-      if (typeof elemObject != 'object' || $.isArray(elemObject)) {
+      if (elemObject == null || typeof elemObject != 'object' || $.isArray(elemObject)) {
         return;
       }
+
       // for each key: fill missing (recursively)
       Object.keys(schemaProps).map(function(key) {
+        if (elemObject[key] !== undefined && key === "foreignKeys" && elemObject[key].length > 0 && elemObject[key][0] !== null) { 
+          // reset the FK field if the fields property is empty to avoid omi parsing errors
+          if (elemObject[key][0]["fields"][0] === null) { 
+            elemObject[key] = []
+          }
+        }
         var prop = schemaProps[key];
         // console.log(path + '.' + key, prop.type)
         if (prop.type == 'object') {
@@ -150,11 +157,13 @@ var MetaEdit = function(config) {
         } else if (prop.type == 'array') {
           elemObject[key] = elemObject[key] || [];
           // if non empty array
-          if ($.isArray(elemObject[key]) && elemObject[key].length > 0) {
+          if ($.isArray(elemObject[key]) && elemObject[key].length > 0 && elemObject[key][0] !== null) {
             elemObject[key].map(function(elem, i) {
               fixRecursive(prop.items.properties, elem, path + '.' + key + '.' + i);
             });
-          }
+          } else {
+            elemObject[key] =[];
+          }          
         } else { // value
           if (elemObject[key] === undefined) {
             // console.log('adding empty value: ' + path + '.' + key)