add retry for readin

embark/dashboard/models.py

@@ -21,14 +21,17 @@ class SoftwareInfo(models.Model):
     """
     Many-to-many object for SBOM entries
     """
-    id = models.UUIDField(primary_key=True, default=uuid.uuid4)
+    id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=True)
+    type = models.CharField(verbose_name="type of blob", blank=False, editable=True, default="NA", max_length=256)
     name = models.CharField(verbose_name="software name", blank=False, editable=True, default="NA", max_length=256)
+    group = models.CharField(verbose_name="grouping", blank=False, editable=True, default="NA", max_length=256)
     version = models.CharField(verbose_name="software version", blank=False, editable=True, default="1.0", max_length=32)
-    hash = models.CharField(verbose_name="identivication hash", blank=False, editable=True, default="NA", max_length=1024)
+    hashes = models.CharField(verbose_name="identivication hash", blank=False, editable=True, default="NA", max_length=1024)
     cpe = models.CharField(verbose_name="CPE identifier", blank=False, editable=True, default="NA", max_length=256)
     type = models.CharField(verbose_name="software type", blank=False, editable=True, default="data", max_length=50)
     purl = models.CharField(verbose_name="PUrl identifier", blank=False, editable=True, default="NA", max_length=256)
-    details = models.JSONField(null=True)
+    description = models.CharField(verbose_name="description", blank=False, editable=True, default="NA", max_length=1024)
+    properties = models.JSONField(null=True, editable=True)
 
 
 class Result(models.Model):

embark/embark/asgi.py

@@ -22,7 +22,7 @@
 
 from embark.routing import ws_urlpatterns
 
-os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'embark.settings.dev')
+os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'embark.settings.deploy')
 
 
 application = ProtocolTypeRouter({

embark/embark/wsgi.py

@@ -14,6 +14,6 @@
 
 from django.core.wsgi import get_wsgi_application
 
-os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'embark.settings.dev')
+os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'embark.settings.deploy')
 
 application = get_wsgi_application()

embark/manage.py

@@ -7,7 +7,7 @@
 
 def main():
     """Run administrative tasks."""
-    os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'embark.settings.dev')
+    os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'embark.settings.deploy')
 
     try:
         # pylint: disable=import-outside-toplevel

embark/porter/forms.py

@@ -27,3 +27,7 @@ class FirmwareAnalysisExportForm(forms.Form):
 
 class DeleteZipForm(forms.Form):
     zip_file = forms.ModelChoiceField(queryset=LogZipFile.objects, empty_label='Select zip-file to delete')
+
+
+class RetryImportForm(forms.Form):
+    analysis = forms.ModelChoiceField(queryset=FirmwareAnalysis.objects.filter(failed=True))

embark/porter/importer.py

@@ -29,8 +29,6 @@ def result_read_in(analysis_id):
     res = None
     csv_directory = f"{settings.EMBA_LOG_ROOT}/{analysis_id}/emba_logs/csv_logs/"
     csv_list = [os.path.join(csv_directory, file_) for file_ in os.listdir(csv_directory)]
-    json_directory = f"{settings.EMBA_LOG_ROOT}/{analysis_id}/emba_logs/json_logs/"
-    json_list = [os.path.join(json_directory, file_) for file_ in os.listdir(json_directory)]
     for file_ in csv_list:
         logger.debug("trying to read: %s", file_)
         if os.path.isfile(file_):      # TODO change check. > if valid EMBA csv file
@@ -43,15 +41,20 @@ def result_read_in(analysis_id):
                 # FIXME f20 in emba is broken!
                 # res = f20_csv(file_, analysis_id)
                 # logger.debug("Result for %s created or updated", analysis_id)
-    for file_ in json_list:
-        logger.debug("trying to read: %s", file_)
-        if os.path.isfile(file_):      # TODO change check. > if valid EMBA json file
-            logger.debug("File %s found and attempting to read", file_)
-            if file_.endswith('f15_cyclonedx_sbom.json'):
-                logger.info("f15 readin for %s skipped", analysis_id)
-                # f21_cyclonedx_sbom_json.json move into db object
-                res = f15_json(file_, analysis_id)
-        # TODO license info etc
+    # json_directory = f"{settings.EMBA_LOG_ROOT}/{analysis_id}/emba_logs/json_logs/"
+    # json_list = [os.path.join(json_directory, file_) for file_ in os.listdir(json_directory)]
+    # for file_ in json_list:
+    #     logger.debug("trying to read: %s", file_)
+    #     if os.path.isfile(file_):      # TODO change check. > if valid EMBA json file
+    #         logger.debug("File %s found and attempting to read", file_)
+    #         if file_.endswith('f15_cyclonedx_sbom.json'):
+    #             logger.info("f15 readin for %s skipped", analysis_id)
+    #             # f21_cyclonedx_sbom_json.json move into db object
+    #             res = f15_json(file_, analysis_id)
+    sbom_file = f"{settings.EMBA_LOG_ROOT}/{analysis_id}/emba_logs/SBOM/EMBA_cyclonedx_sbom.json"
+    if os.path.isfile(sbom_file):
+        logger.debug("File %s found and attempting to read", sbom_file)
+        res = f15_json(sbom_file, analysis_id)
     return res
 
 
@@ -214,23 +217,27 @@ def f15_json(_file_path, _analysis_id):
     )
     with open(_file_path, 'r', encoding='utf-8') as f15_json_file:
         f15_data = json.load(f15_json_file)
-        for component_ in f15_data.components:
+        for component_ in f15_data['components']:
+            logger.debug("Component is %s", component_)
             try:
                 new_sbom, add_ = SoftwareInfo.objects.update_or_create(
-                    name=component_.name,
-                    version=component_.version,
-                    hashes=component_.hashes,
-                    cpe=component_.cpe,
-                    type=component_.type,
-                    purl=component_.purl,
-                    details=component_
+                    id=component_['bom-ref'],
+                    name=component_['name'],
+                    type=component_['type'],
+                    group=component_['group'] or 'NA',
+                    version=component_['version'] or 'NA',
+                    hashes=component_['hashes'],
+                    cpe=component_['cpe'] or 'NA',
+                    purl=component_['purl'] or 'NA',
+                    description=component_['description'] or 'NA',
+                    properties=component_['properties'] or 'NA'
                 )
+                logger.debug("Was new? %s", add_)
                 logger.debug("Adding SBOM item: %s to res %s", new_sbom, res)
                 if add_:
                     res.sbom.add(new_sbom)
             except builtins.Exception as error_:
                 logger.error("Error in f15 readin: %s", error_)
-                logger.error("Component is %s ; Was new? %s", component_, add_)
     logger.debug("read f15 json done")
     return res
 

embark/porter/urls.py

@@ -16,7 +16,7 @@
 
     path('export/', views.export_menu, name='embark-export-menu'),
     path('export/download/', views.export_analysis, name='embark-export-analysis'),
-    path('export/zip/<uuid:analysis_id>/', views.make_zip, name='embark-make-zip')
-
+    path('export/zip/<uuid:analysis_id>/', views.make_zip, name='embark-make-zip'),
 
+    path('retry-import/', views.retry_import, name='embark-retry-import'),
 ]

embark/porter/views.py

@@ -19,8 +19,9 @@
 from uploader.forms import DeviceForm, LabelForm, VendorForm
 from uploader.models import FirmwareAnalysis
 from porter.exporter import result_json
+from porter.importer import result_read_in
 from porter.models import LogZipFile
-from porter.forms import FirmwareAnalysisImportForm, FirmwareAnalysisExportForm, DeleteZipForm
+from porter.forms import FirmwareAnalysisImportForm, FirmwareAnalysisExportForm, DeleteZipForm, RetryImportForm
 
 
 logger = logging.getLogger(__name__)
@@ -30,6 +31,7 @@
 @login_required(login_url='/' + settings.LOGIN_URL)
 @require_http_methods(["GET"])
 def import_menu(request):
+    retry_form = RetryImportForm()
     import_read_form = FirmwareAnalysisImportForm()
     device_form = DeviceForm()
     vendor_form = VendorForm()
@@ -38,7 +40,7 @@ def import_menu(request):
         delete_form = DeleteZipForm(initial={'zip-file': LogZipFile.objects.latest('upload_date')})
     else:
         delete_form = DeleteZipForm()
-    return render(request, 'porter/import.html', {'import_read_form': import_read_form, 'device_form': device_form, 'vendor_form': vendor_form, 'label_form': label_form, 'delete_form': delete_form})
+    return render(request, 'porter/import.html', {'import_read_form': import_read_form, 'device_form': device_form, 'vendor_form': vendor_form, 'label_form': label_form, 'delete_form': delete_form, 'retry_form': retry_form})
 
 
 @login_required(login_url='/' + settings.LOGIN_URL)
@@ -191,3 +193,38 @@ def make_zip(request, analysis_id):
     except FirmwareAnalysis.DoesNotExist:
         messages.error(request, 'No analysis with that id found')
     return redirect('embark-ReportDashboard')
+
+
+@login_required(login_url='/' + settings.LOGIN_URL)
+@require_http_methods(["POST"])
+def retry_import(request):
+    """
+    retry to read emba_log foler with the importer
+
+        returns:
+        messages, status
+    """
+    req_logger.info("%s requested with: %s", __name__, request)
+    form = RetryImportForm(request.POST)
+    if form.is_valid():
+        logger.debug("Posted Form is valid")
+        analysis = form.cleaned_data['analysis']
+        if analysis.user != request.user:
+            logger.error("Permission denied - %s", request)
+            messages.error(request, "You don't have permission")
+            return redirect('..')
+        # trying to re-import
+        logger.info("Importing analysis with %s", analysis.id)
+        # TODO
+        if result_read_in(analysis.id) is not None:
+            analysis.finished = True
+            analysis.failed = False
+            analysis.save(update_fields=["finished", "failed"])
+            # success
+            logger.info("Successfully submitted for re-import %s", analysis.id)
+            messages.info(request, 'import submitted for ' + str(analysis.id))
+            return redirect('..')
+        messages.error(request, 're-import failed')
+        return redirect('..')
+    messages.error(request, 'form invalid')
+    return HttpResponseBadRequest("invalid form")

embark/templates/porter/import.html

@@ -21,6 +21,7 @@
         </div>
         <div class="col-sm">
             {% block read %}{% include "porter/read.html" %}{% endblock read %}
+            {% block retry %}{% include "porter/retry.html" %}{% endblock retry %}
         </div>
 
     </div>

embark/templates/porter/retry.html

@@ -0,0 +1,18 @@
+{% load django_bootstrap5 %}
+
+        <div class="col-sm">
+            <div class="box">
+                <p class="mainText">Retry-importing</p>
+                <p class="subText">Manually</p>
+
+                <form action="{% url 'embark-retry-import' %}" method="post" id="form">
+                    {% csrf_token %}
+                    <div class="innerBlock">
+                        {% load filters %}
+                        {% bootstrap_form retry_form %}
+                        {% bootstrap_button "Re-import" button_type="submit" button_class="btn-primary" %}
+                    </div>
+                </form>
+
+            </div>
+        </div>