diff --git a/embark/dashboard/models.py b/embark/dashboard/models.py index 066218d8..be3052c6 100644 --- a/embark/dashboard/models.py +++ b/embark/dashboard/models.py @@ -21,14 +21,17 @@ class SoftwareInfo(models.Model): """ Many-to-many object for SBOM entries """ - id = models.UUIDField(primary_key=True, default=uuid.uuid4) + id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=True) + type = models.CharField(verbose_name="type of blob", blank=False, editable=True, default="NA", max_length=256) name = models.CharField(verbose_name="software name", blank=False, editable=True, default="NA", max_length=256) + group = models.CharField(verbose_name="grouping", blank=False, editable=True, default="NA", max_length=256) version = models.CharField(verbose_name="software version", blank=False, editable=True, default="1.0", max_length=32) - hash = models.CharField(verbose_name="identivication hash", blank=False, editable=True, default="NA", max_length=1024) + hashes = models.CharField(verbose_name="identivication hash", blank=False, editable=True, default="NA", max_length=1024) cpe = models.CharField(verbose_name="CPE identifier", blank=False, editable=True, default="NA", max_length=256) type = models.CharField(verbose_name="software type", blank=False, editable=True, default="data", max_length=50) purl = models.CharField(verbose_name="PUrl identifier", blank=False, editable=True, default="NA", max_length=256) - details = models.JSONField(null=True) + description = models.CharField(verbose_name="description", blank=False, editable=True, default="NA", max_length=1024) + properties = models.JSONField(null=True, editable=True) class Result(models.Model): diff --git a/embark/embark/asgi.py b/embark/embark/asgi.py index 878a8389..f227b603 100644 --- a/embark/embark/asgi.py +++ b/embark/embark/asgi.py @@ -22,7 +22,7 @@ from embark.routing import ws_urlpatterns -os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'embark.settings.dev') +os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'embark.settings.deploy') application = ProtocolTypeRouter({ diff --git a/embark/embark/wsgi.py b/embark/embark/wsgi.py index f4021295..3f78c856 100644 --- a/embark/embark/wsgi.py +++ b/embark/embark/wsgi.py @@ -14,6 +14,6 @@ from django.core.wsgi import get_wsgi_application -os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'embark.settings.dev') +os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'embark.settings.deploy') application = get_wsgi_application() diff --git a/embark/manage.py b/embark/manage.py index 1b1aa9a8..582ed7e7 100755 --- a/embark/manage.py +++ b/embark/manage.py @@ -7,7 +7,7 @@ def main(): """Run administrative tasks.""" - os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'embark.settings.dev') + os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'embark.settings.deploy') try: # pylint: disable=import-outside-toplevel diff --git a/embark/porter/forms.py b/embark/porter/forms.py index 67a25a0d..c5b698ff 100644 --- a/embark/porter/forms.py +++ b/embark/porter/forms.py @@ -27,3 +27,7 @@ class FirmwareAnalysisExportForm(forms.Form): class DeleteZipForm(forms.Form): zip_file = forms.ModelChoiceField(queryset=LogZipFile.objects, empty_label='Select zip-file to delete') + + +class RetryImportForm(forms.Form): + analysis = forms.ModelChoiceField(queryset=FirmwareAnalysis.objects.filter(failed=True)) diff --git a/embark/porter/importer.py b/embark/porter/importer.py index 7b58ea08..c4753f59 100644 --- a/embark/porter/importer.py +++ b/embark/porter/importer.py @@ -29,8 +29,6 @@ def result_read_in(analysis_id): res = None csv_directory = f"{settings.EMBA_LOG_ROOT}/{analysis_id}/emba_logs/csv_logs/" csv_list = [os.path.join(csv_directory, file_) for file_ in os.listdir(csv_directory)] - json_directory = f"{settings.EMBA_LOG_ROOT}/{analysis_id}/emba_logs/json_logs/" - json_list = [os.path.join(json_directory, file_) for file_ in os.listdir(json_directory)] for file_ in csv_list: logger.debug("trying to read: %s", file_) if os.path.isfile(file_): # TODO change check. > if valid EMBA csv file @@ -43,15 +41,20 @@ def result_read_in(analysis_id): # FIXME f20 in emba is broken! # res = f20_csv(file_, analysis_id) # logger.debug("Result for %s created or updated", analysis_id) - for file_ in json_list: - logger.debug("trying to read: %s", file_) - if os.path.isfile(file_): # TODO change check. > if valid EMBA json file - logger.debug("File %s found and attempting to read", file_) - if file_.endswith('f15_cyclonedx_sbom.json'): - logger.info("f15 readin for %s skipped", analysis_id) - # f21_cyclonedx_sbom_json.json move into db object - res = f15_json(file_, analysis_id) - # TODO license info etc + # json_directory = f"{settings.EMBA_LOG_ROOT}/{analysis_id}/emba_logs/json_logs/" + # json_list = [os.path.join(json_directory, file_) for file_ in os.listdir(json_directory)] + # for file_ in json_list: + # logger.debug("trying to read: %s", file_) + # if os.path.isfile(file_): # TODO change check. > if valid EMBA json file + # logger.debug("File %s found and attempting to read", file_) + # if file_.endswith('f15_cyclonedx_sbom.json'): + # logger.info("f15 readin for %s skipped", analysis_id) + # # f21_cyclonedx_sbom_json.json move into db object + # res = f15_json(file_, analysis_id) + sbom_file = f"{settings.EMBA_LOG_ROOT}/{analysis_id}/emba_logs/SBOM/EMBA_cyclonedx_sbom.json" + if os.path.isfile(sbom_file): + logger.debug("File %s found and attempting to read", sbom_file) + res = f15_json(sbom_file, analysis_id) return res @@ -214,23 +217,27 @@ def f15_json(_file_path, _analysis_id): ) with open(_file_path, 'r', encoding='utf-8') as f15_json_file: f15_data = json.load(f15_json_file) - for component_ in f15_data.components: + for component_ in f15_data['components']: + logger.debug("Component is %s", component_) try: new_sbom, add_ = SoftwareInfo.objects.update_or_create( - name=component_.name, - version=component_.version, - hashes=component_.hashes, - cpe=component_.cpe, - type=component_.type, - purl=component_.purl, - details=component_ + id=component_['bom-ref'], + name=component_['name'], + type=component_['type'], + group=component_['group'] or 'NA', + version=component_['version'] or 'NA', + hashes=component_['hashes'], + cpe=component_['cpe'] or 'NA', + purl=component_['purl'] or 'NA', + description=component_['description'] or 'NA', + properties=component_['properties'] or 'NA' ) + logger.debug("Was new? %s", add_) logger.debug("Adding SBOM item: %s to res %s", new_sbom, res) if add_: res.sbom.add(new_sbom) except builtins.Exception as error_: logger.error("Error in f15 readin: %s", error_) - logger.error("Component is %s ; Was new? %s", component_, add_) logger.debug("read f15 json done") return res diff --git a/embark/porter/urls.py b/embark/porter/urls.py index da9e31ae..82f07104 100644 --- a/embark/porter/urls.py +++ b/embark/porter/urls.py @@ -16,7 +16,7 @@ path('export/', views.export_menu, name='embark-export-menu'), path('export/download/', views.export_analysis, name='embark-export-analysis'), - path('export/zip//', views.make_zip, name='embark-make-zip') - + path('export/zip//', views.make_zip, name='embark-make-zip'), + path('retry-import/', views.retry_import, name='embark-retry-import'), ] diff --git a/embark/porter/views.py b/embark/porter/views.py index dc043546..a8d992d5 100644 --- a/embark/porter/views.py +++ b/embark/porter/views.py @@ -19,8 +19,9 @@ from uploader.forms import DeviceForm, LabelForm, VendorForm from uploader.models import FirmwareAnalysis from porter.exporter import result_json +from porter.importer import result_read_in from porter.models import LogZipFile -from porter.forms import FirmwareAnalysisImportForm, FirmwareAnalysisExportForm, DeleteZipForm +from porter.forms import FirmwareAnalysisImportForm, FirmwareAnalysisExportForm, DeleteZipForm, RetryImportForm logger = logging.getLogger(__name__) @@ -30,6 +31,7 @@ @login_required(login_url='/' + settings.LOGIN_URL) @require_http_methods(["GET"]) def import_menu(request): + retry_form = RetryImportForm() import_read_form = FirmwareAnalysisImportForm() device_form = DeviceForm() vendor_form = VendorForm() @@ -38,7 +40,7 @@ def import_menu(request): delete_form = DeleteZipForm(initial={'zip-file': LogZipFile.objects.latest('upload_date')}) else: delete_form = DeleteZipForm() - return render(request, 'porter/import.html', {'import_read_form': import_read_form, 'device_form': device_form, 'vendor_form': vendor_form, 'label_form': label_form, 'delete_form': delete_form}) + return render(request, 'porter/import.html', {'import_read_form': import_read_form, 'device_form': device_form, 'vendor_form': vendor_form, 'label_form': label_form, 'delete_form': delete_form, 'retry_form': retry_form}) @login_required(login_url='/' + settings.LOGIN_URL) @@ -191,3 +193,38 @@ def make_zip(request, analysis_id): except FirmwareAnalysis.DoesNotExist: messages.error(request, 'No analysis with that id found') return redirect('embark-ReportDashboard') + + +@login_required(login_url='/' + settings.LOGIN_URL) +@require_http_methods(["POST"]) +def retry_import(request): + """ + retry to read emba_log foler with the importer + + returns: + messages, status + """ + req_logger.info("%s requested with: %s", __name__, request) + form = RetryImportForm(request.POST) + if form.is_valid(): + logger.debug("Posted Form is valid") + analysis = form.cleaned_data['analysis'] + if analysis.user != request.user: + logger.error("Permission denied - %s", request) + messages.error(request, "You don't have permission") + return redirect('..') + # trying to re-import + logger.info("Importing analysis with %s", analysis.id) + # TODO + if result_read_in(analysis.id) is not None: + analysis.finished = True + analysis.failed = False + analysis.save(update_fields=["finished", "failed"]) + # success + logger.info("Successfully submitted for re-import %s", analysis.id) + messages.info(request, 'import submitted for ' + str(analysis.id)) + return redirect('..') + messages.error(request, 're-import failed') + return redirect('..') + messages.error(request, 'form invalid') + return HttpResponseBadRequest("invalid form") diff --git a/embark/templates/porter/import.html b/embark/templates/porter/import.html index 603dfaad..77f77a22 100644 --- a/embark/templates/porter/import.html +++ b/embark/templates/porter/import.html @@ -21,6 +21,7 @@
{% block read %}{% include "porter/read.html" %}{% endblock read %} + {% block retry %}{% include "porter/retry.html" %}{% endblock retry %}
diff --git a/embark/templates/porter/retry.html b/embark/templates/porter/retry.html new file mode 100644 index 00000000..d7712499 --- /dev/null +++ b/embark/templates/porter/retry.html @@ -0,0 +1,18 @@ +{% load django_bootstrap5 %} + +
+
+

Retry-importing

+

Manually

+ +
+ {% csrf_token %} +
+ {% load filters %} + {% bootstrap_form retry_form %} + {% bootstrap_button "Re-import" button_type="submit" button_class="btn-primary" %} +
+
+ +
+
\ No newline at end of file