feat(sast): Filter js files generate by ts (#6220)

* fix integration tests

* .

* ?

* .

* filter js generated by ts

* continue

* add more cases

* lint

* mypy

* .

* add try

* .

* .

* fix path

* fix test

checkov/sast/engines/files_filter_manager.py

@@ -0,0 +1,86 @@
+import logging
+import os
+import json
+from typing import Set, List, Dict
+
+from checkov.common.sast.consts import SastLanguages
+
+
+class FilesFilterManager:
+    def __init__(self, source_codes: List[str], languages: Set[SastLanguages]) -> None:
+        self.source_codes: List[str] = source_codes
+        self.languages: Set[SastLanguages] = languages
+
+    def get_files_to_filter(self) -> List[str]:
+        files_to_filter: List[str] = []
+        try:
+            if SastLanguages.JAVASCRIPT in self.languages:
+                files_to_filter += self._get_js_files_to_filter()
+        except Exception as e:
+            logging.debug(f'Error filtering js files generated by ts: {e}')
+        return files_to_filter
+
+    def _get_js_files_to_filter(self) -> List[str]:
+        js_files_to_filter = []
+
+        for path in self.source_codes:
+            js_files: List[Dict[str, str]] = []
+            ts_files: List[Dict[str, str]] = []
+            tsconfig_files: List[Dict[str, str]] = []
+            for (dirpath, _, filenames) in os.walk(path):
+                if '/node_modules/' in dirpath:
+                    continue
+                for filename in filenames:
+                    if filename.endswith('.ts'):
+                        ts_files.append({'full_path': os.sep.join([dirpath, filename]), 'dir': dirpath, 'name': filename})
+                    if filename.endswith('tsconfig.json'):
+                        tsconfig_files.append({'full_path': os.sep.join([dirpath, filename]), 'dir': dirpath, 'name': filename})
+                    if filename.endswith('.js'):
+                        js_files.append({'full_path': os.sep.join([dirpath, filename]), 'dir': dirpath, 'name': filename})
+
+            js_files_to_filter += FilesFilterManager._filter_by_tsconfig(tsconfig_files)
+            js_files_to_filter += FilesFilterManager._filter_direct_build_js(js_files, ts_files, js_files_to_filter)
+
+        return js_files_to_filter
+
+    @staticmethod
+    def _filter_direct_build_js(js_files: List[Dict[str, str]], ts_files: List[Dict[str, str]], filtered_by_tsconfig: List[str]) -> List[str]:
+        js_files_to_filter: List[str] = []
+        for js_file in js_files:
+            js_dir = js_file.get('dir', '')
+            already_skipped = False
+            for filtered_by_tsconfig_path in filtered_by_tsconfig:
+                if js_dir.startswith(filtered_by_tsconfig_path):
+                    already_skipped = True
+                    break
+            if already_skipped:
+                continue
+            for ts_file in ts_files:
+                if ts_file.get('dir', '') == js_dir and ts_file.get('name', '')[:-3] == js_file.get('name', '')[:-3]:
+                    js_files_to_filter.append(js_file.get('full_path', ''))
+                    break
+        return js_files_to_filter
+
+    @staticmethod
+    def _filter_by_tsconfig(tsconfig_files: List[Dict[str, str]]) -> List[str]:
+        js_files_to_filter: List[str] = []
+        for tsconfig_file in tsconfig_files:
+            with open(tsconfig_file.get('full_path', '')) as fp:
+                config = json.load(fp)
+            out_dir = config.get('compilerOptions', {}).get('outDir')
+            out_file = config.get('compilerOptions', {}).get('outFile')
+            if out_dir:
+                build_dir = out_dir
+            elif out_file:
+                build_dir = out_file
+            else:
+                build_dir = tsconfig_file.get('dir')
+
+            # relative path
+            if not build_dir.startswith('/'):
+                build_path = os.path.abspath(tsconfig_file.get('dir', '') + '/' + build_dir)
+            # absolute path
+            else:
+                build_path = build_dir
+            js_files_to_filter.append(build_path)
+        return js_files_to_filter

checkov/sast/engines/prisma_engine.py

@@ -32,6 +32,7 @@
 from checkov.sast.record import SastRecord
 from checkov.sast.report import SastReport
 from checkov.cdk.report import CDKReport
+from checkov.sast.engines.files_filter_manager import FilesFilterManager
 
 logger = logging.getLogger(__name__)
 
@@ -84,6 +85,11 @@ def get_reports(self, targets: List[str], registry: Registry, languages: Set[Sas
 
         check_threshold, skip_check_threshold = self.get_check_thresholds(registry)
 
+        skip_paths = registry.runner_filter.excluded_paths if registry.runner_filter else []
+
+        files_filter_manager = FilesFilterManager(targets, languages)
+        skip_paths += files_filter_manager.get_files_to_filter()
+
         library_input: LibraryInput = {
             'languages': languages,
             'source_codes': targets,
@@ -92,7 +98,7 @@ def get_reports(self, targets: List[str], registry: Registry, languages: Set[Sas
             'skip_checks': registry.runner_filter.skip_checks if registry.runner_filter else [],
             'check_threshold': check_threshold,
             'skip_check_threshold': skip_check_threshold,
-            'skip_path': registry.runner_filter.excluded_paths if registry.runner_filter else [],
+            'skip_path': skip_paths,
             'report_imports': registry.runner_filter.report_sast_imports if registry.runner_filter else False,
             'remove_default_policies': registry.runner_filter.remove_default_sast_policies if registry.runner_filter else False,
             'report_reachability': registry.runner_filter.report_sast_reachability if registry.runner_filter else False,

tests/sast/source_code/js_filtered_build_ts/example1/app.ts

@@ -0,0 +1,2 @@
+let message: string = 'Hello, World!';
+console.log(message);

tests/sast/source_code/js_filtered_build_ts/example1/tsconfig.json

@@ -0,0 +1,13 @@
+{
+  "compilerOptions": {
+    "module": "system",
+    "noImplicitAny": true,
+    "removeComments": true,
+    "preserveConstEnums": true,
+    "outDir": "./build",
+    "sourceMap": true
+  },
+  "include": [
+    "./*.ts"
+  ]
+}

tests/sast/source_code/js_filtered_build_ts/example2/app.ts

@@ -0,0 +1,2 @@
+let message: string = 'Hello, World!';
+console.log(message);

tests/sast/source_code/js_filtered_build_ts/example2/tsconfig.json

@@ -0,0 +1,13 @@
+{
+  "compilerOptions": {
+    "module": "system",
+    "noImplicitAny": true,
+    "removeComments": true,
+    "preserveConstEnums": true,
+    "outFile": "./build/file.js",
+    "sourceMap": true
+  },
+  "include": [
+    "./*.ts"
+  ]
+}

tests/sast/test_filter_files_manager.py

@@ -0,0 +1,14 @@
+from checkov.sast.engines.files_filter_manager import FilesFilterManager
+from checkov.common.sast.consts import SastLanguages
+import pathlib
+import os
+
+
+def test_sast_js_filtered_files_by_ts():
+    test_dir = os.path.join(pathlib.Path(__file__).parent.resolve(), 'source_code', 'js_filtered_build_ts')
+    files_filter_manager = FilesFilterManager([test_dir], set([SastLanguages.JAVASCRIPT]))
+    filtered_paths = files_filter_manager.get_files_to_filter()
+    assert len(filtered_paths) == 3
+    assert filtered_paths[0].endswith('example2/build/file.js')
+    assert filtered_paths[1].endswith('example1/build')
+    assert filtered_paths[2].endswith('example3/main.js')