From 8b54ca0021d5cb6536ebaa32fc2ba623cb59f731 Mon Sep 17 00:00:00 2001 From: Garrett Marconet Date: Tue, 7 Nov 2023 09:57:38 -0800 Subject: [PATCH 01/20] add timeout --- .../plugins/veracode_sbom/settings.json | 3 +- backend/engine/utils/plugin.py | 40 ++++++++++++++----- 2 files changed, 33 insertions(+), 10 deletions(-) diff --git a/backend/engine/plugins/veracode_sbom/settings.json b/backend/engine/plugins/veracode_sbom/settings.json index befc11fb..be4d24e6 100644 --- a/backend/engine/plugins/veracode_sbom/settings.json +++ b/backend/engine/plugins/veracode_sbom/settings.json @@ -2,5 +2,6 @@ "name": "Veracode SBOM", "type": "sbom", "image": "$ECR/artemis/veracode:latest", - "enabled": "$ARTEMIS_FEATURE_VERACODE_ENABLED" + "enabled": "$ARTEMIS_FEATURE_VERACODE_ENABLED", + "timeout": 1 } \ No newline at end of file diff --git a/backend/engine/utils/plugin.py b/backend/engine/utils/plugin.py index ec45dc4f..46c95b56 100644 --- a/backend/engine/utils/plugin.py +++ b/backend/engine/utils/plugin.py @@ -52,6 +52,7 @@ class Result: alerts: list debug: list disabled: bool = False + timedout: bool = False @dataclass @@ -61,6 +62,7 @@ class PluginSettings: name: str plugin_type: str feature: str + timeout: int def get_engine_vars(scan, depth=None, include_dev=False, services=None): @@ -175,6 +177,7 @@ def get_plugin_settings(plugin: str) -> PluginSettings: name=settings.get("name"), plugin_type=settings.get("type", "misc"), feature=settings.get("feature"), + timeout=settings.get("timeout"), ) @@ -292,8 +295,23 @@ def run_plugin(plugin, scan, scan_images, depth=None, include_dev=False, feature scan, settings.image, plugin, depth, include_dev, scan_images, plugin_config, services ) - # Run the plugin inside the settings.image - r = subprocess.run(plugin_command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, check=False) + try: + # Run the plugin inside the settings.image + r = subprocess.run( + plugin_command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, check=False, timeout=settings.timeout + ) + except subprocess.TimeoutExpired: + return Result( + name=settings.name, + type=settings.plugin_type, + success=False, + truncated=False, + details=[], + errors=[], + alerts=[], + debug=[f"Plugin {settings.name} timed out"], + timedout=True, + ) log.info("--- Plugin log start ---\n%s", r.stderr.decode("utf-8").strip()) # Inject plugin logs log.info("--- Plugin log end ---") @@ -330,13 +348,8 @@ def run_plugin(plugin, scan, scan_images, depth=None, include_dev=False, feature ) except json.JSONDecodeError: - err_str = r.stdout.decode("utf-8") or "" - if "Traceback" in r.stderr.decode("utf-8"): - # If the plugin output contains a stack trace include the last line in the error - last_line = r.stderr.decode("utf-8").strip().split("\n")[-1] - err_str += f" [Error: {last_line}]" - err = f"Plugin returned invalid output: {err_str}" - log.error(err) + err_str = _get_error_str(r.stdout.decode("utf-8")) + log.error(f"Plugin returned invalid output: {err_str}") return Result( name=settings.name, type=settings.plugin_type, @@ -349,6 +362,15 @@ def run_plugin(plugin, scan, scan_images, depth=None, include_dev=False, feature ) +def _get_error_str(output): + err_str = output or "" + if "Traceback" in output: + # If the plugin output contains a stack trace include the last line in the error + last_line = output.strip().split("\n")[-1] + err_str += f" [Error: {last_line}]" + return err_str + + def process_event_info(scan, results, plugin_type, plugin_name): log.info("Processing event info") timestamp = get_iso_timestamp() From 635239573040cbdf8fa1cfa37b5c18e26f84f51e Mon Sep 17 00:00:00 2001 From: Garrett Marconet Date: Wed, 8 Nov 2023 10:37:37 -0800 Subject: [PATCH 02/20] minimize api changes; do not implement timeout for veracode_sbom --- backend/engine/plugins/veracode_sbom/settings.json | 5 ++--- backend/engine/utils/plugin.py | 6 ++---- 2 files changed, 4 insertions(+), 7 deletions(-) diff --git a/backend/engine/plugins/veracode_sbom/settings.json b/backend/engine/plugins/veracode_sbom/settings.json index be4d24e6..0d88d124 100644 --- a/backend/engine/plugins/veracode_sbom/settings.json +++ b/backend/engine/plugins/veracode_sbom/settings.json @@ -2,6 +2,5 @@ "name": "Veracode SBOM", "type": "sbom", "image": "$ECR/artemis/veracode:latest", - "enabled": "$ARTEMIS_FEATURE_VERACODE_ENABLED", - "timeout": 1 -} \ No newline at end of file + "enabled": "$ARTEMIS_FEATURE_VERACODE_ENABLED" +} diff --git a/backend/engine/utils/plugin.py b/backend/engine/utils/plugin.py index 46c95b56..9afebe56 100644 --- a/backend/engine/utils/plugin.py +++ b/backend/engine/utils/plugin.py @@ -52,7 +52,6 @@ class Result: alerts: list debug: list disabled: bool = False - timedout: bool = False @dataclass @@ -307,10 +306,9 @@ def run_plugin(plugin, scan, scan_images, depth=None, include_dev=False, feature success=False, truncated=False, details=[], - errors=[], + errors=[f"Plugin {settings.name} timed out. (>={settings.timeout} seconds)"], alerts=[], - debug=[f"Plugin {settings.name} timed out"], - timedout=True, + debug=[], ) log.info("--- Plugin log start ---\n%s", r.stderr.decode("utf-8").strip()) # Inject plugin logs From 59626027ea802f4880751f0628a759fa78f789db Mon Sep 17 00:00:00 2001 From: Garrett Marconet Date: Wed, 8 Nov 2023 13:17:26 -0800 Subject: [PATCH 03/20] remove changes that are no longer relevant --- .../engine/plugins/veracode_sbom/settings.json | 2 +- backend/engine/utils/plugin.py | 18 +++++++----------- 2 files changed, 8 insertions(+), 12 deletions(-) diff --git a/backend/engine/plugins/veracode_sbom/settings.json b/backend/engine/plugins/veracode_sbom/settings.json index 0d88d124..befc11fb 100644 --- a/backend/engine/plugins/veracode_sbom/settings.json +++ b/backend/engine/plugins/veracode_sbom/settings.json @@ -3,4 +3,4 @@ "type": "sbom", "image": "$ECR/artemis/veracode:latest", "enabled": "$ARTEMIS_FEATURE_VERACODE_ENABLED" -} +} \ No newline at end of file diff --git a/backend/engine/utils/plugin.py b/backend/engine/utils/plugin.py index 9afebe56..b065b903 100644 --- a/backend/engine/utils/plugin.py +++ b/backend/engine/utils/plugin.py @@ -346,8 +346,13 @@ def run_plugin(plugin, scan, scan_images, depth=None, include_dev=False, feature ) except json.JSONDecodeError: - err_str = _get_error_str(r.stdout.decode("utf-8")) - log.error(f"Plugin returned invalid output: {err_str}") + err_str = r.stdout.decode("utf-8") or "" + if "Traceback" in r.stderr.decode("utf-8"): + # If the plugin output contains a stack trace include the last line in the error + last_line = r.stderr.decode("utf-8").strip().split("\n")[-1] + err_str += f" [Error: {last_line}]" + err = f"Plugin returned invalid output: {err_str}" + log.error(err) return Result( name=settings.name, type=settings.plugin_type, @@ -360,15 +365,6 @@ def run_plugin(plugin, scan, scan_images, depth=None, include_dev=False, feature ) -def _get_error_str(output): - err_str = output or "" - if "Traceback" in output: - # If the plugin output contains a stack trace include the last line in the error - last_line = output.strip().split("\n")[-1] - err_str += f" [Error: {last_line}]" - return err_str - - def process_event_info(scan, results, plugin_type, plugin_name): log.info("Processing event info") timestamp = get_iso_timestamp() From db2a3a8a8ee1636bdb7b1864891511c06bfdfa6d Mon Sep 17 00:00:00 2001 From: Garrett Marconet Date: Wed, 8 Nov 2023 13:19:02 -0800 Subject: [PATCH 04/20] change timeout error language --- backend/engine/utils/plugin.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/engine/utils/plugin.py b/backend/engine/utils/plugin.py index b065b903..e2c13dcc 100644 --- a/backend/engine/utils/plugin.py +++ b/backend/engine/utils/plugin.py @@ -306,7 +306,7 @@ def run_plugin(plugin, scan, scan_images, depth=None, include_dev=False, feature success=False, truncated=False, details=[], - errors=[f"Plugin {settings.name} timed out. (>={settings.timeout} seconds)"], + errors=[f"Plugin {settings.name} exceeded maximum runtime ({settings.timeout} seconds)."], alerts=[], debug=[], ) From a07b1a35cef815ef375652912ca3989904e2f202 Mon Sep 17 00:00:00 2001 From: Garrett Marconet Date: Wed, 8 Nov 2023 13:24:48 -0800 Subject: [PATCH 05/20] add docs on timeout --- backend/engine/plugins/README.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/backend/engine/plugins/README.md b/backend/engine/plugins/README.md index eb49bf33..74ab068d 100644 --- a/backend/engine/plugins/README.md +++ b/backend/engine/plugins/README.md @@ -37,13 +37,15 @@ This file contains the plugin's settings. The settings are: - type: The type of the plugin: `vulnerability`, `secrets`, `static_analysis`, `inventory`, `configuration`, `sbom` - image: The Docker image the plugin runs in. If using a non-public image prefix with `$ECR`. During local development `$ECR` is ignored. When running in AWS, the `$ECR` environment variable is populated with the current account's ECR URL. - disabled: Boolean. If true the plugin is skipped. This can also be a string containing the name of an environment variable containing the boolean value, for example "$ARTEMIS_FEATURE_XYZ_ENABLED". +- timeout: Integer. The amount of time, in seconds, to allow the plugin to run before exiting early with an error. ```json { "name": "Example", "type": null, "image": "$ECR/:", - "disabled": true + "disabled": true, + "timeout": 3600 } ``` From 4dd947c1f69592d60e0f5c3498372ba2bcd59045 Mon Sep 17 00:00:00 2001 From: Garrett Marconet Date: Wed, 8 Nov 2023 13:35:23 -0800 Subject: [PATCH 06/20] imply optional nature of timeout --- backend/engine/plugins/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/engine/plugins/README.md b/backend/engine/plugins/README.md index 74ab068d..a3d89fda 100644 --- a/backend/engine/plugins/README.md +++ b/backend/engine/plugins/README.md @@ -37,7 +37,7 @@ This file contains the plugin's settings. The settings are: - type: The type of the plugin: `vulnerability`, `secrets`, `static_analysis`, `inventory`, `configuration`, `sbom` - image: The Docker image the plugin runs in. If using a non-public image prefix with `$ECR`. During local development `$ECR` is ignored. When running in AWS, the `$ECR` environment variable is populated with the current account's ECR URL. - disabled: Boolean. If true the plugin is skipped. This can also be a string containing the name of an environment variable containing the boolean value, for example "$ARTEMIS_FEATURE_XYZ_ENABLED". -- timeout: Integer. The amount of time, in seconds, to allow the plugin to run before exiting early with an error. +- timeout: Integer. If set, this is the amount of time, in seconds, to allow the plugin to run before exiting early with an error. ```json { From b2ec348e728798524348ab50eed4ec85d441a43e Mon Sep 17 00:00:00 2001 From: Garrett Marconet Date: Tue, 14 Nov 2023 15:07:10 -0800 Subject: [PATCH 07/20] save errors from sbom scans and return them --- .../plugins/veracode_sbom/settings.json | 5 +-- backend/engine/processor/processor.py | 19 +++++++---- .../json_report/json_report/report.py | 7 ++++ .../json_report/json_report/results/sbom.py | 32 +++++++++++++++++++ .../artemisdb/artemisdb/artemisdb/consts.py | 1 + 5 files changed, 55 insertions(+), 9 deletions(-) create mode 100644 backend/lambdas/generators/json_report/json_report/results/sbom.py diff --git a/backend/engine/plugins/veracode_sbom/settings.json b/backend/engine/plugins/veracode_sbom/settings.json index befc11fb..ef362972 100644 --- a/backend/engine/plugins/veracode_sbom/settings.json +++ b/backend/engine/plugins/veracode_sbom/settings.json @@ -1,6 +1,7 @@ -{ +t{ "name": "Veracode SBOM", "type": "sbom", "image": "$ECR/artemis/veracode:latest", - "enabled": "$ARTEMIS_FEATURE_VERACODE_ENABLED" + "enabled": "$ARTEMIS_FEATURE_VERACODE_ENABLED", + "timeout": 28800 } \ No newline at end of file diff --git a/backend/engine/processor/processor.py b/backend/engine/processor/processor.py index d35dc8e4..eb7011cb 100644 --- a/backend/engine/processor/processor.py +++ b/backend/engine/processor/processor.py @@ -145,14 +145,19 @@ def process_plugins(self, images: dict, services: dict) -> None: logger.info("Plugin %s is disabled", plugin) else: logger.info("Plugin %s completed, updating results", plugin) - if results.type != "sbom": - if results.type == "vulnerability": - process_vulns(results, self.scan.get_scan_object(), plugin) - self.scan.create_plugin_result_set(start_time, results) - self._cache_results(results) - else: - # Process SBOM results + + if results.type == "sbom": process_sbom(results, self.scan.get_scan_object()) + + # SBOM results should not be returned directly in the scan, so clear details + results.details = [] + + elif results.type == "vulnerability": + process_vulns(results, self.scan.get_scan_object(), plugin) + + self.scan.create_plugin_result_set(start_time, results) + self._cache_results(results) + logger.info("Plugin %s results updated", plugin) # Clean and reset the repo in case the plugin wrote any files to disk. This way files created or diff --git a/backend/lambdas/generators/json_report/json_report/report.py b/backend/lambdas/generators/json_report/json_report/report.py index 866a9cd6..0d7c30ac 100644 --- a/backend/lambdas/generators/json_report/json_report/report.py +++ b/backend/lambdas/generators/json_report/json_report/report.py @@ -2,6 +2,7 @@ from json_report.results.configuration import get_configuration from json_report.results.inventory import get_inventory from json_report.results.results import PLUGIN_RESULTS, PluginErrors +from json_report.results.sbom import get_sbom from json_report.results.secret import get_secrets from json_report.results.static_analysis import get_static_analysis from json_report.results.vuln import get_vulns @@ -16,6 +17,7 @@ def get_report(scan_id, params=None): vuln_results = PLUGIN_RESULTS({}, PluginErrors(), True, None) secret_results = PLUGIN_RESULTS({}, PluginErrors(), True, None) + sbom_results = PLUGIN_RESULTS({}, PluginErrors(), True, None) sa_results = PLUGIN_RESULTS({}, PluginErrors(), True, None) inv_results = PLUGIN_RESULTS({}, PluginErrors(), True, None) config_results = PLUGIN_RESULTS({}, PluginErrors(), True, None) @@ -35,6 +37,9 @@ def get_report(scan_id, params=None): if "results" not in params or "secrets" in params["results"]: secret_results = get_secrets(scan, params) + if "results" not in params or "sbom" in params["results"]: + sbom_results = get_sbom(scan) + if "results" not in params or "static_analysis" in params["results"]: sa_results = get_static_analysis(scan, params) @@ -46,6 +51,7 @@ def get_report(scan_id, params=None): errors.update(vuln_results.errors) errors.update(secret_results.errors) + errors.update(sbom_results.errors) errors.update(sa_results.errors) errors.update(inv_results.errors) errors.update(config_results.errors) @@ -53,6 +59,7 @@ def get_report(scan_id, params=None): success = ( vuln_results.success and secret_results.success + and sbom_results.success and sa_results.success and inv_results.success and config_results.success diff --git a/backend/lambdas/generators/json_report/json_report/results/sbom.py b/backend/lambdas/generators/json_report/json_report/results/sbom.py new file mode 100644 index 00000000..64650041 --- /dev/null +++ b/backend/lambdas/generators/json_report/json_report/results/sbom.py @@ -0,0 +1,32 @@ +from artemisdb.artemisdb.consts import PluginType +from artemisdb.artemisdb.models import Scan +from json_report.results.results import PLUGIN_RESULTS, PluginErrors +from json_report.util.const import WL_CONFIGURATION_KEYS +from json_report.util.util import dict_eq + + +def get_sbom(scan: Scan) -> PLUGIN_RESULTS: + """ + Unify the output of sbom plugins. We deliberately omit the `details` property in the database + (since SBOM results are stored in s3), so this returns no scan results or summary + NOTE: unit tests are located at api/tests/test_generate_report.py + Inspect tests for expected output format. + :param scan: django object of Artemis repo scan + :return: dictionary of None for findings, list of errors, boolean success, and None for summary + """ + + errors = PluginErrors() + + plugin = _empty = object() + for plugin in scan.pluginresult_set.filter(plugin_type=PluginType.SBOM.value): + errors.update(plugin) + + return PLUGIN_RESULTS(None, errors, True, None) + + +def allowlisted_configuration(item, allow_list): + for al_item in allow_list: + if dict_eq(al_item.value, item, WL_CONFIGURATION_KEYS): + + return True + return False diff --git a/backend/libs/artemisdb/artemisdb/artemisdb/consts.py b/backend/libs/artemisdb/artemisdb/artemisdb/consts.py index b0b2a355..cb2acc7f 100644 --- a/backend/libs/artemisdb/artemisdb/artemisdb/consts.py +++ b/backend/libs/artemisdb/artemisdb/artemisdb/consts.py @@ -14,6 +14,7 @@ class PluginType(Enum): CONFIGURATION = "configuration" INVENTORY = "inventory" VULN = "vulnerability" + SBOM = "sbom" SECRETS = "secrets" STATIC_ANALYSIS = "static_analysis" From def1b7786070f74d6c8e39816eb62093eeda6c6d Mon Sep 17 00:00:00 2001 From: Garrett Marconet Date: Tue, 14 Nov 2023 15:18:42 -0800 Subject: [PATCH 08/20] add sbom test --- .../json_report/tests/test_generate_report.py | 30 +++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/backend/lambdas/generators/json_report/tests/test_generate_report.py b/backend/lambdas/generators/json_report/tests/test_generate_report.py index a9560652..5592962b 100644 --- a/backend/lambdas/generators/json_report/tests/test_generate_report.py +++ b/backend/lambdas/generators/json_report/tests/test_generate_report.py @@ -5,6 +5,7 @@ from json_report.results.configuration import get_configuration from json_report.results.inventory import get_inventory from json_report.results.results import PLUGIN_RESULTS, PluginErrors +from json_report.results.sbom import get_sbom from json_report.results.static_analysis import get_static_analysis from json_report.util.const import DEFAULT_SCAN_QUERY_PARAMS @@ -246,6 +247,18 @@ end_time=datetime(year=2020, month=2, day=19, hour=15, minute=1, second=55, tzinfo=timezone.utc), ) +TEST_VERACODE_SBOM = PluginResult( + plugin_name="Veracode SBOM", + plugin_type="sbom", + success=True, + details=[], + errors=["test error"], + alerts=["test alert"], + debug=["test debug"], + start_time=datetime(year=2020, month=2, day=19, hour=15, minute=1, second=54, tzinfo=timezone.utc), + end_time=datetime(year=2020, month=2, day=19, hour=15, minute=1, second=55, tzinfo=timezone.utc), +) + class TestGenerateReport(unittest.TestCase): def test_get_static_analysis_report_for_brakeman(self): @@ -449,6 +462,23 @@ def test_get_configuration_report_for_github_repo_health(self): mock_scan.pluginresult_set.filter.return_value = [TEST_GITHUB_REPO_HEALTH] configuration = get_configuration(mock_scan, DEFAULT_SCAN_QUERY_PARAMS) self.assertEqual(expected_configuration, configuration) + + def test_get_sbom_report_for_veracode_sbom(self): + expected_sbom = PluginResult( + None, + PluginErrors( + errors=["test error"], + alerts=["test alert"], + debug=["test debug"], + ), + True, + None, + ) + + mock_scan = unittest.mock.MagicMock(side_effect=Scan()) + mock_scan.pluginresult_set.filter.return_value = [TEST_VERACODE_SBOM] + sbom = get_sbom(mock_scan) + self.assertEqual(expected_sbom, sbom) def test_get_static_analysis_report_diff(self): expected_report = PLUGIN_RESULTS( From cd43a8bb0bb74f4c6161660ea33f73c9f2a50ac7 Mon Sep 17 00:00:00 2001 From: Garrett Marconet Date: Tue, 14 Nov 2023 15:23:30 -0800 Subject: [PATCH 09/20] remove erroneous character --- backend/engine/plugins/veracode_sbom/settings.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/engine/plugins/veracode_sbom/settings.json b/backend/engine/plugins/veracode_sbom/settings.json index ef362972..6d4510fa 100644 --- a/backend/engine/plugins/veracode_sbom/settings.json +++ b/backend/engine/plugins/veracode_sbom/settings.json @@ -1,4 +1,4 @@ -t{ +{ "name": "Veracode SBOM", "type": "sbom", "image": "$ECR/artemis/veracode:latest", From 63df9d75ae5da2e93d54ff2672937641322a6adb Mon Sep 17 00:00:00 2001 From: Garrett Marconet Date: Wed, 15 Nov 2023 14:12:35 -0800 Subject: [PATCH 10/20] add db migration --- .../migrations/0047_auto_20231115_1408.py | 43 +++++++++++++++++++ 1 file changed, 43 insertions(+) create mode 100644 backend/libs/artemisdb/artemisdb/artemisdb/migrations/0047_auto_20231115_1408.py diff --git a/backend/libs/artemisdb/artemisdb/artemisdb/migrations/0047_auto_20231115_1408.py b/backend/libs/artemisdb/artemisdb/artemisdb/migrations/0047_auto_20231115_1408.py new file mode 100644 index 00000000..d0add6b1 --- /dev/null +++ b/backend/libs/artemisdb/artemisdb/artemisdb/migrations/0047_auto_20231115_1408.py @@ -0,0 +1,43 @@ +# Generated by Django 3.2.23 on 2023-11-15 14:08 + +import artemisdb.artemisdb.consts +from django.db import migrations, models + + +class Migration(migrations.Migration): + dependencies = [ + ("artemisdb", "0046_component_component_type"), + ] + + operations = [ + migrations.AlterField( + model_name="plugin", + name="type", + field=models.CharField( + choices=[ + (artemisdb.artemisdb.consts.PluginType["CONFIGURATION"], "configuration"), + (artemisdb.artemisdb.consts.PluginType["INVENTORY"], "inventory"), + (artemisdb.artemisdb.consts.PluginType["VULN"], "vulnerability"), + (artemisdb.artemisdb.consts.PluginType["SBOM"], "sbom"), + (artemisdb.artemisdb.consts.PluginType["SECRETS"], "secrets"), + (artemisdb.artemisdb.consts.PluginType["STATIC_ANALYSIS"], "static_analysis"), + ], + max_length=64, + ), + ), + migrations.AlterField( + model_name="pluginresult", + name="plugin_type", + field=models.CharField( + choices=[ + (artemisdb.artemisdb.consts.PluginType["CONFIGURATION"], "configuration"), + (artemisdb.artemisdb.consts.PluginType["INVENTORY"], "inventory"), + (artemisdb.artemisdb.consts.PluginType["VULN"], "vulnerability"), + (artemisdb.artemisdb.consts.PluginType["SBOM"], "sbom"), + (artemisdb.artemisdb.consts.PluginType["SECRETS"], "secrets"), + (artemisdb.artemisdb.consts.PluginType["STATIC_ANALYSIS"], "static_analysis"), + ], + max_length=64, + ), + ), + ] From ab6a5bb9b1ed9fa6035b9d46e05c9c8aaceb19dd Mon Sep 17 00:00:00 2001 From: Garrett Marconet Date: Wed, 15 Nov 2023 14:31:31 -0800 Subject: [PATCH 11/20] reduce veracode_sbom timeout to 1s to test out reporting in nonprod --- backend/engine/plugins/veracode_sbom/settings.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/engine/plugins/veracode_sbom/settings.json b/backend/engine/plugins/veracode_sbom/settings.json index 6d4510fa..be4d24e6 100644 --- a/backend/engine/plugins/veracode_sbom/settings.json +++ b/backend/engine/plugins/veracode_sbom/settings.json @@ -3,5 +3,5 @@ "type": "sbom", "image": "$ECR/artemis/veracode:latest", "enabled": "$ARTEMIS_FEATURE_VERACODE_ENABLED", - "timeout": 28800 + "timeout": 1 } \ No newline at end of file From 7acdad2afa64a0e1d64fe1c0a62f5925d5c1e98c Mon Sep 17 00:00:00 2001 From: Garrett Marconet Date: Wed, 15 Nov 2023 15:00:36 -0800 Subject: [PATCH 12/20] return veracode sbom timeout to 8hrs --- backend/engine/plugins/veracode_sbom/settings.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/engine/plugins/veracode_sbom/settings.json b/backend/engine/plugins/veracode_sbom/settings.json index be4d24e6..6d4510fa 100644 --- a/backend/engine/plugins/veracode_sbom/settings.json +++ b/backend/engine/plugins/veracode_sbom/settings.json @@ -3,5 +3,5 @@ "type": "sbom", "image": "$ECR/artemis/veracode:latest", "enabled": "$ARTEMIS_FEATURE_VERACODE_ENABLED", - "timeout": 1 + "timeout": 28800 } \ No newline at end of file From 18cb8bd8bd8a2139c3b2ca9bbc319b14f27f893a Mon Sep 17 00:00:00 2001 From: Garrett Marconet Date: Wed, 15 Nov 2023 15:07:37 -0800 Subject: [PATCH 13/20] remove unused code --- .../generators/json_report/json_report/results/sbom.py | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/backend/lambdas/generators/json_report/json_report/results/sbom.py b/backend/lambdas/generators/json_report/json_report/results/sbom.py index 64650041..7e5dad93 100644 --- a/backend/lambdas/generators/json_report/json_report/results/sbom.py +++ b/backend/lambdas/generators/json_report/json_report/results/sbom.py @@ -1,8 +1,6 @@ from artemisdb.artemisdb.consts import PluginType from artemisdb.artemisdb.models import Scan from json_report.results.results import PLUGIN_RESULTS, PluginErrors -from json_report.util.const import WL_CONFIGURATION_KEYS -from json_report.util.util import dict_eq def get_sbom(scan: Scan) -> PLUGIN_RESULTS: @@ -22,11 +20,3 @@ def get_sbom(scan: Scan) -> PLUGIN_RESULTS: errors.update(plugin) return PLUGIN_RESULTS(None, errors, True, None) - - -def allowlisted_configuration(item, allow_list): - for al_item in allow_list: - if dict_eq(al_item.value, item, WL_CONFIGURATION_KEYS): - - return True - return False From aac0f6c3f39a597d3e55434d1b352224f0f5e0c1 Mon Sep 17 00:00:00 2001 From: Garrett Marconet Date: Wed, 15 Nov 2023 15:20:44 -0800 Subject: [PATCH 14/20] fix named tuple --- .../generators/json_report/tests/test_generate_report.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/backend/lambdas/generators/json_report/tests/test_generate_report.py b/backend/lambdas/generators/json_report/tests/test_generate_report.py index 5592962b..2982a106 100644 --- a/backend/lambdas/generators/json_report/tests/test_generate_report.py +++ b/backend/lambdas/generators/json_report/tests/test_generate_report.py @@ -467,9 +467,9 @@ def test_get_sbom_report_for_veracode_sbom(self): expected_sbom = PluginResult( None, PluginErrors( - errors=["test error"], - alerts=["test alert"], - debug=["test debug"], + ["test error"], + ["test alert"], + ["test debug"], ), True, None, From bfefa2ffa75d1a048b7f3e494f88d52c19601181 Mon Sep 17 00:00:00 2001 From: Garrett Marconet Date: Wed, 15 Nov 2023 15:27:40 -0800 Subject: [PATCH 15/20] use PLUGIN_RESULTS --- .../generators/json_report/tests/test_generate_report.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/lambdas/generators/json_report/tests/test_generate_report.py b/backend/lambdas/generators/json_report/tests/test_generate_report.py index 2982a106..5fce67fa 100644 --- a/backend/lambdas/generators/json_report/tests/test_generate_report.py +++ b/backend/lambdas/generators/json_report/tests/test_generate_report.py @@ -464,7 +464,7 @@ def test_get_configuration_report_for_github_repo_health(self): self.assertEqual(expected_configuration, configuration) def test_get_sbom_report_for_veracode_sbom(self): - expected_sbom = PluginResult( + expected_sbom = PLUGIN_RESULTS( None, PluginErrors( ["test error"], From 47027ee0004c7f6c7a9a4f807a28ce197371fea6 Mon Sep 17 00:00:00 2001 From: Garrett Marconet Date: Wed, 15 Nov 2023 15:43:16 -0800 Subject: [PATCH 16/20] fix test --- .../json_report/tests/test_generate_report.py | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/backend/lambdas/generators/json_report/tests/test_generate_report.py b/backend/lambdas/generators/json_report/tests/test_generate_report.py index 5fce67fa..44590bf3 100644 --- a/backend/lambdas/generators/json_report/tests/test_generate_report.py +++ b/backend/lambdas/generators/json_report/tests/test_generate_report.py @@ -252,9 +252,9 @@ plugin_type="sbom", success=True, details=[], - errors=["test error"], - alerts=["test alert"], - debug=["test debug"], + errors=[], + alerts=[], + debug=[], start_time=datetime(year=2020, month=2, day=19, hour=15, minute=1, second=54, tzinfo=timezone.utc), end_time=datetime(year=2020, month=2, day=19, hour=15, minute=1, second=55, tzinfo=timezone.utc), ) @@ -462,15 +462,11 @@ def test_get_configuration_report_for_github_repo_health(self): mock_scan.pluginresult_set.filter.return_value = [TEST_GITHUB_REPO_HEALTH] configuration = get_configuration(mock_scan, DEFAULT_SCAN_QUERY_PARAMS) self.assertEqual(expected_configuration, configuration) - + def test_get_sbom_report_for_veracode_sbom(self): expected_sbom = PLUGIN_RESULTS( None, - PluginErrors( - ["test error"], - ["test alert"], - ["test debug"], - ), + PluginErrors(), True, None, ) From 0009a85e628f575f5222a6dd360b1d49564c2899 Mon Sep 17 00:00:00 2001 From: Garrett Marconet Date: Mon, 20 Nov 2023 09:47:29 -0800 Subject: [PATCH 17/20] add errors to veracode test --- .../json_report/tests/test_generate_report.py | 32 ++++++++++++++++--- 1 file changed, 28 insertions(+), 4 deletions(-) diff --git a/backend/lambdas/generators/json_report/tests/test_generate_report.py b/backend/lambdas/generators/json_report/tests/test_generate_report.py index 44590bf3..139c1949 100644 --- a/backend/lambdas/generators/json_report/tests/test_generate_report.py +++ b/backend/lambdas/generators/json_report/tests/test_generate_report.py @@ -247,14 +247,18 @@ end_time=datetime(year=2020, month=2, day=19, hour=15, minute=1, second=55, tzinfo=timezone.utc), ) +SBOM_ERROR_MSG = "test error" +SBOM_ALERT_MSG = "test alert" +SBOM_DEBUG_MSG = "test debug message" + TEST_VERACODE_SBOM = PluginResult( plugin_name="Veracode SBOM", plugin_type="sbom", success=True, details=[], - errors=[], - alerts=[], - debug=[], + errors=[SBOM_ERROR_MSG], + alerts=[SBOM_ALERT_MSG], + debug=[SBOM_DEBUG_MSG], start_time=datetime(year=2020, month=2, day=19, hour=15, minute=1, second=54, tzinfo=timezone.utc), end_time=datetime(year=2020, month=2, day=19, hour=15, minute=1, second=55, tzinfo=timezone.utc), ) @@ -466,7 +470,9 @@ def test_get_configuration_report_for_github_repo_health(self): def test_get_sbom_report_for_veracode_sbom(self): expected_sbom = PLUGIN_RESULTS( None, - PluginErrors(), + get_plugin_errors( + TEST_VERACODE_SBOM.plugin_name, errors=[SBOM_ERROR_MSG], alerts=[SBOM_ALERT_MSG], debug=[SBOM_DEBUG_MSG] + ), True, None, ) @@ -474,6 +480,14 @@ def test_get_sbom_report_for_veracode_sbom(self): mock_scan = unittest.mock.MagicMock(side_effect=Scan()) mock_scan.pluginresult_set.filter.return_value = [TEST_VERACODE_SBOM] sbom = get_sbom(mock_scan) + print("expected") + print(expected_sbom.errors.errors) + print(expected_sbom.errors.alerts) + print(expected_sbom.errors.debug) + print("real") + print(sbom.errors.errors) + print(sbom.errors.alerts) + print(sbom.errors.debug) self.assertEqual(expected_sbom, sbom) def test_get_static_analysis_report_diff(self): @@ -519,5 +533,15 @@ def run_static_analysis(self, scan, expected_report): self.assertEqual(expected_report, report) +def get_plugin_errors(name, errors, alerts, debug): + plugin_errors = PluginErrors() + + plugin_errors.errors[name] = errors + plugin_errors.alerts[name] = alerts + plugin_errors.debug[name] = debug + + return plugin_errors + + if __name__ == "__main__": unittest.main() From 058925f15403c6cd79d70b42d798056d6fe2d361 Mon Sep 17 00:00:00 2001 From: Garrett Marconet Date: Mon, 20 Nov 2023 09:48:08 -0800 Subject: [PATCH 18/20] use enum values for comparison --- backend/engine/processor/processor.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/backend/engine/processor/processor.py b/backend/engine/processor/processor.py index eb7011cb..9b2f290f 100644 --- a/backend/engine/processor/processor.py +++ b/backend/engine/processor/processor.py @@ -146,13 +146,13 @@ def process_plugins(self, images: dict, services: dict) -> None: else: logger.info("Plugin %s completed, updating results", plugin) - if results.type == "sbom": + if results.type == PluginType.SBOM.value: process_sbom(results, self.scan.get_scan_object()) # SBOM results should not be returned directly in the scan, so clear details results.details = [] - elif results.type == "vulnerability": + elif results.type == PluginType.VULN.value: process_vulns(results, self.scan.get_scan_object(), plugin) self.scan.create_plugin_result_set(start_time, results) From 46d2b0e6dde0ff4804f80603ba21f8c436c6c7b2 Mon Sep 17 00:00:00 2001 From: Garrett Marconet Date: Mon, 20 Nov 2023 09:50:56 -0800 Subject: [PATCH 19/20] fix test path in comment --- .../generators/json_report/json_report/results/configuration.py | 2 +- .../lambdas/generators/json_report/json_report/results/sbom.py | 2 +- .../json_report/json_report/results/static_analysis.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/backend/lambdas/generators/json_report/json_report/results/configuration.py b/backend/lambdas/generators/json_report/json_report/results/configuration.py index d2ce7b26..aa5259a4 100644 --- a/backend/lambdas/generators/json_report/json_report/results/configuration.py +++ b/backend/lambdas/generators/json_report/json_report/results/configuration.py @@ -11,7 +11,7 @@ def get_configuration(scan: Scan, params: dict) -> PLUGIN_RESULTS: """ Unify the output of configuration plugins - NOTE: unit tests are located at api/tests/test_generate_report.py + NOTE: unit tests are located at backend/lambdas/generators/json_report/tests/test_generate_report.py Inspect tests for expected output format. :param scan: django object of Artemis repo scan :param params: Only used for filtering severity diff --git a/backend/lambdas/generators/json_report/json_report/results/sbom.py b/backend/lambdas/generators/json_report/json_report/results/sbom.py index 7e5dad93..fa2e1da7 100644 --- a/backend/lambdas/generators/json_report/json_report/results/sbom.py +++ b/backend/lambdas/generators/json_report/json_report/results/sbom.py @@ -7,7 +7,7 @@ def get_sbom(scan: Scan) -> PLUGIN_RESULTS: """ Unify the output of sbom plugins. We deliberately omit the `details` property in the database (since SBOM results are stored in s3), so this returns no scan results or summary - NOTE: unit tests are located at api/tests/test_generate_report.py + NOTE: unit tests are located at backend/lambdas/generators/json_report/tests/test_generate_report.py Inspect tests for expected output format. :param scan: django object of Artemis repo scan :return: dictionary of None for findings, list of errors, boolean success, and None for summary diff --git a/backend/lambdas/generators/json_report/json_report/results/static_analysis.py b/backend/lambdas/generators/json_report/json_report/results/static_analysis.py index 82eae806..53dd6302 100644 --- a/backend/lambdas/generators/json_report/json_report/results/static_analysis.py +++ b/backend/lambdas/generators/json_report/json_report/results/static_analysis.py @@ -13,7 +13,7 @@ def get_static_analysis(scan: Scan, params: dict) -> PLUGIN_RESULTS: """ Unify the output of static analysis plugins - NOTE: unit tests are located at api/tests/test_generate_report.py + NOTE: unit tests are located at backend/lambdas/generators/json_report/tests/test_generate_report.py Inspect tests for expected output format. :param scan: django object of Artemis repo scan :param params: Only used for filtering severity From 8d4adce78b3ecbbd7aa5575c323cb9fa300e8b27 Mon Sep 17 00:00:00 2001 From: Garrett Marconet Date: Mon, 20 Nov 2023 09:53:08 -0800 Subject: [PATCH 20/20] add explanation for success being hardcoded to True --- .../generators/json_report/json_report/results/sbom.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/backend/lambdas/generators/json_report/json_report/results/sbom.py b/backend/lambdas/generators/json_report/json_report/results/sbom.py index fa2e1da7..22dfc997 100644 --- a/backend/lambdas/generators/json_report/json_report/results/sbom.py +++ b/backend/lambdas/generators/json_report/json_report/results/sbom.py @@ -6,16 +6,19 @@ def get_sbom(scan: Scan) -> PLUGIN_RESULTS: """ Unify the output of sbom plugins. We deliberately omit the `details` property in the database - (since SBOM results are stored in s3), so this returns no scan results or summary + (since SBOM results are stored in s3), so this returns no scan results or summary. This will + always return True for `success`, since SBOM scans are just an inventory (and the existence of a + dependancy cannot "fail" the scan) and the plugin itself failing is not a reason to fail the + larger scan NOTE: unit tests are located at backend/lambdas/generators/json_report/tests/test_generate_report.py Inspect tests for expected output format. :param scan: django object of Artemis repo scan - :return: dictionary of None for findings, list of errors, boolean success, and None for summary + :return: dictionary of None for findings, list of errors, True for success, and None for summary """ errors = PluginErrors() - plugin = _empty = object() + plugin = object() for plugin in scan.pluginresult_set.filter(plugin_type=PluginType.SBOM.value): errors.update(plugin)