[do not merge] feat: latest dev updates split

.pre-commit-config.yaml

@@ -1,6 +1,6 @@
 repos:
     - repo: https://github.com/pre-commit/pre-commit-hooks
-      rev: v3.3.0
+      rev: v4.6.0
       hooks:
           - id: check-builtin-literals
             args: ["--no-allow-dict-kwargs"]
@@ -11,33 +11,34 @@ repos:
           - id: double-quote-string-fixer
           - id: end-of-file-fixer
           - id: name-tests-test
+            exclude: "^tests/testing/.*$"
           - id: trailing-whitespace
     - repo: https://github.com/pycqa/flake8
-      rev: 3.9.2
+      rev: 7.0.0
       hooks:
           - id: flake8
             args: ["--max-line-length", "100"]
             exclude: ^test_data/|bumpity.py$
     - repo: https://github.com/asottile/reorder_python_imports
-      rev: v2.3.6
+      rev: v3.12.0
       hooks:
           - id: reorder-python-imports
             language_version: python3
             exclude: bumpity.py$
     - repo: https://github.com/asottile/add-trailing-comma
-      rev: v2.0.1
+      rev: v3.1.0
       hooks:
           - id: add-trailing-comma
     - repo: https://github.com/pre-commit/mirrors-autopep8
-      rev: v1.5.4
+      rev: v2.0.4
       hooks:
           - id: autopep8
     - repo: https://github.com/ibm/detect-secrets
       # If you desire to use a specific version of detect-secrets, you can replace `master` with other git revisions such as branch, tag or commit sha.
       # You are encouraged to use static refs such as tags, instead of branch name
       #
       # Running "pre-commit autoupdate" would automatically updates rev to latest tag
-      rev: 0.13.1+ibm.58.dss
+      rev: 0.13.1+ibm.62.dss
       hooks:
           - id: detect-secrets # pragma: whitelist secret
             # Add options for detect-secrets-hook binary. You can run `detect-secrets-hook --help` to list out all possible options.

.secrets.baseline

@@ -3,7 +3,7 @@
     "files": "test_data/.*|tests/.*|^.secrets.baseline$",
     "lines": null
   },
-  "generated_at": "2023-05-25T14:44:17Z",
+  "generated_at": "2024-05-06T12:15:33Z",
   "plugins_used": [
     {
       "name": "AWSKeyDetector"
@@ -242,7 +242,7 @@
       }
     ]
   },
-  "version": "0.13.1+ibm.58.dss",
+  "version": "0.13.1+ibm.62.dss",
   "word_list": {
     "file": null,
     "hash": null

Makefile.ibm

@@ -61,8 +61,9 @@ trivy-scan-python-vulnerabilities:
 	PIPENV_IGNORE_VIRTUALENVS=True # Forces Pipenv to ignore the existing virtual environment and create its own instead
 
 	# Generate a Pipfile.lock, Trivy does not auto-detect requirements-dev.txt (https://aquasecurity.github.io/trivy/v0.28.1/docs/vulnerability/detection/language/)
-	./scripts/gen-pipfile.sh > Pipfile
+	#./scripts/gen-pipfile.sh > Pipfile
 	pipenv --python `which python3`
+	pipenv install -r requirements-dev.txt
 	pipenv lock
 	$(TRIVY) fs --exit-code 1 --ignore-unfixed --scanners vuln ./
 

detect_secrets/__init__.py

@@ -1 +1 @@
-VERSION = '0.13.1+ibm.62.dss'
+VERSION = '0.13.1+ibm.63.dss'

detect_secrets/core/baseline.py

@@ -16,6 +16,7 @@
 def initialize(
     path,
     plugins,
+    plugins_reuse_excludes=None,
     exclude_files_regex=None,
     exclude_lines_regex=None,
     word_list_file=None,
@@ -33,6 +34,9 @@ def initialize(
     :type plugins: tuple of detect_secrets.plugins.base.BasePlugin
     :param plugins: rules to initialize the SecretsCollection with.
 
+    :type plugins_reuse_excludes: bool|None
+    :param plugins_reuse_excludes optional bool whether plugins were forced to reuse excludes.
+
     :type exclude_files_regex: str|None
     :type exclude_lines_regex: str|None
 
@@ -53,6 +57,7 @@ def initialize(
     """
     output = SecretsCollection(
         plugins,
+        plugins_reuse_excludes=plugins_reuse_excludes,
         exclude_files=exclude_files_regex,
         exclude_lines=exclude_lines_regex,
         word_list_file=word_list_file,
@@ -361,22 +366,30 @@ def _get_git_tracked_files(rootdir='.'):
     :returns: filepaths to files which git currently tracks (locally)
     """
     output = []
+
+    # git <1.8.5 https://github.com/git/git/commit/44e1e4d67d5148c245db362cc48c3cc6c2ec82ca
+    # doesn't support -C <path> and we can achieve the same using cwd arg to subproc
+    cmd = ['git', 'ls-files']
+    if not os.path.exists(rootdir) or not os.path.isdir(rootdir):
+        log.debug(f'Skipping {rootdir} bc dir doesn\'t exist or isn\'t a directory')
+        return []
+
     try:
         with open(os.devnull, 'w') as fnull:
-            git_files = subprocess.check_output(
-                [
-                    'git',
-                    '-C', rootdir,
-                    'ls-files',
-                ],
-                stderr=fnull,
-            )
+            git_files = subprocess.check_output(cmd, cwd=rootdir, stderr=fnull)
+
         for filename in git_files.decode('utf-8').split():
             relative_path = util.get_relative_path_if_in_cwd(rootdir, filename)
             if relative_path:
                 output.append(relative_path)
-    except subprocess.CalledProcessError:
+
+    except subprocess.CalledProcessError as err:
+        log.error(
+            'detect-secrets: Encountered error trying to list git tracked ' +
+            f'files for dir {rootdir}: {str(err)}',
+        )
         pass
+
     return output
 
 

detect_secrets/core/secrets_collection.py

@@ -21,6 +21,7 @@ class SecretsCollection:
     def __init__(
         self,
         plugins=(),
+        plugins_reuse_excludes=None,
         exclude_files=None,
         exclude_lines=None,
         word_list_file=None,
@@ -46,6 +47,7 @@ def __init__(
         """
         self.data = {}
         self.plugins = plugins
+        self.plugins_reuse_excludes = plugins_reuse_excludes
         self.exclude_files = exclude_files
         self.exclude_lines = exclude_lines
         self.word_list_file = word_list_file
@@ -342,6 +344,7 @@ def format_for_baseline_output(self):
         plugins_used = sorted(plugins_used, key=lambda x: x['name'])
 
         return {
+            **({'plugins_reuse_excludes': True} if self.plugins_reuse_excludes else {}),
             'generated_at': strftime('%Y-%m-%dT%H:%M:%SZ', gmtime()),
             'exclude': {
                 'files': self.exclude_files,

detect_secrets/core/usage.py

@@ -5,6 +5,14 @@
 from detect_secrets.constants import DEFAULT_GHE_INSTANCE
 
 
+def add_plugins_reuse_excludes_flag(parser):
+    parser.add_argument(
+        '--plugins-reuse-excludes',
+        action='store_true',
+        help='Force plugins to try re-using existing exclude contents.',
+    )
+
+
 def add_exclude_lines_argument(parser):
     parser.add_argument(
         '--exclude-lines',
@@ -89,6 +97,7 @@ def add_default_arguments(self):
 
     def add_pre_commit_arguments(self):
         self._add_filenames_argument()\
+            ._add_plugins_reuse_excludes_flag()\
             ._add_set_baseline_argument()\
             ._add_exclude_lines_argument()\
             ._add_word_list_argument()\
@@ -154,6 +163,10 @@ def _add_set_baseline_argument(self):
         )
         return self
 
+    def _add_plugins_reuse_excludes_flag(self):
+        add_plugins_reuse_excludes_flag(self.parser)
+        return self
+
     def _add_exclude_lines_argument(self):
         add_exclude_lines_argument(self.parser)
         return self
@@ -222,6 +235,10 @@ def _add_initialize_baseline_argument(self):
             ),
         )
 
+        # Pairing `--plugins-reuse-excludes` to
+        # both pre-commit and `--scan` because it can be used for both.
+        add_plugins_reuse_excludes_flag(self.parser)
+
         # Pairing `--exclude-lines` and `--word-list` to
         # both pre-commit and `--scan` because it can be used for both.
         add_exclude_lines_argument(self.parser)

detect_secrets/main.py

@@ -17,6 +17,20 @@ def parse_args(argv, parserBuilder):
     return parserBuilder.add_console_use_arguments().parse_args(argv)
 
 
+def maybe_get_existing_exclude(exclude_files, exclude_lines, old_baseline):
+    if not old_baseline:
+        return exclude_files, exclude_lines
+
+    previously_included = old_baseline.get('exclude', None)
+    if not previously_included:
+        return exclude_files, exclude_lines
+
+    files = '|'.join(filter(None, (exclude_files, previously_included.get('files', None))))
+    lines = '|'.join(filter(None, (exclude_lines, previously_included.get('lines', None))))
+
+    return files, lines
+
+
 def main(argv=None):
     if len(sys.argv) == 1:  # pragma: no cover
         sys.argv.append('-h')
@@ -36,6 +50,10 @@ def main(argv=None):
         if args.word_list_file:
             automaton, word_list_hash = build_automaton(args.word_list_file)
 
+        _baseline = _get_existing_baseline(args.import_filename, args.string)
+        if args.plugins_reuse_excludes or (_baseline and _baseline.get('plugins_reuse_excludes', False)):  # noqa: E501
+            args.exclude_files, args.exclude_lines = maybe_get_existing_exclude(args.exclude_files, args.exclude_lines, _baseline)  # noqa: E501
+
         # Plugins are *always* rescanned with fresh settings, because
         # we want to get the latest updates.
         plugins = initialize.from_parser_builder(
@@ -153,7 +171,7 @@ def _perform_scan(args, plugins, automaton, word_list_hash):
 
     :rtype: dict
     """
-    old_baseline = _get_existing_baseline(args.import_filename)
+    old_baseline = _get_existing_baseline(args.import_filename, args.string)
     if old_baseline:
         plugins = initialize.merge_plugins_from_baseline(
             _get_plugins_from_baseline(old_baseline, tuple(args.plugin_filenames)),
@@ -173,13 +191,17 @@ def _perform_scan(args, plugins, automaton, word_list_hash):
         if not args.word_list_file and old_baseline.get('word_list'):
             args.word_list_file = old_baseline['word_list']['file']
 
+        if not args.plugins_reuse_excludes:
+            args.plugins_reuse_excludes = old_baseline.get('plugins_reuse_excludes', False)
+
     # If we have knowledge of an existing baseline file, we should use
     # that knowledge and add it to our exclude_files regex.
     if args.import_filename:
         _add_baseline_to_exclude_files(args)
 
     new_baseline = baseline.initialize(
         plugins=plugins,
+        plugins_reuse_excludes=args.plugins_reuse_excludes,
         exclude_files_regex=args.exclude_files,
         exclude_lines_regex=args.exclude_lines,
         word_list_file=args.word_list_file,
@@ -200,7 +222,7 @@ def _perform_scan(args, plugins, automaton, word_list_hash):
     return new_baseline
 
 
-def _get_existing_baseline(import_filename):
+def _get_existing_baseline(import_filename, args_string):
     # Favors --update argument over stdin.
     if import_filename:
         try:
@@ -214,7 +236,8 @@ def _get_existing_baseline(import_filename):
                     file=sys.stderr,
                 )
                 raise fnf_error
-    if not sys.stdin.isatty():
+
+    if not sys.stdin.isatty() and not args_string:
         stdin = sys.stdin.read().strip()
         if stdin:
             return json.loads(stdin)

detect_secrets/plugins/artifactory.py

@@ -21,7 +21,7 @@ class ArtifactoryDetector(RegexBasedDetector):
 
     def verify(self, token, *args, **kwargs):
         try:
-            if type(token) == bytes:
+            if isinstance(token, bytes):
                 token = token.decode('UTF-8')
             headers = {'X-JFrog-Art-API': token}
             response = requests.get(

detect_secrets/plugins/base.py

@@ -59,7 +59,7 @@ def __init__(
         exclude_lines_regex=None,
         should_verify=False,
         false_positive_heuristics=None,
-        **kwargs
+        **kwargs,
     ):
         """
         :type exclude_lines_regex: str|None
@@ -426,7 +426,7 @@ def secret_generator(  # lgtm [py/inheritance/incorrect-overridden-signature]
         self,
         string,
         *args,
-        **kwargs
+        **kwargs,
     ):
         for regex in self.denylist:
             for match in regex.findall(string):

detect_secrets/plugins/common/initialize.py

@@ -41,7 +41,7 @@ def from_parser_builder(
                 automaton=automaton,
                 should_verify_secrets=should_verify_secrets,
                 plugin_filenames=plugin_filenames,
-                **plugins_dict[plugin_name]
+                **plugins_dict[plugin_name],
             ),
         )
 
@@ -200,7 +200,7 @@ def from_plugin_classname(
             exclude_lines_regex=exclude_lines_regex,
             automaton=automaton,
             should_verify=should_verify_secrets,
-            **kwargs
+            **kwargs,
         )
     except TypeError:
         log.warning('Unable to initialize plugin!')
@@ -248,5 +248,5 @@ def from_secret_type(secret_type, settings, plugin_filenames=None):
                 automaton=None,
                 should_verify_secrets=False,
 
-                **plugin_init_vars
+                **plugin_init_vars,
             )

detect_secrets/plugins/github_enterprise.py

@@ -77,7 +77,7 @@ def __init__(self, ghe_instance=DEFAULT_GHE_INSTANCE, *args, **kwargs):
 
     def verify(self, token, *args, **kwargs):
         try:
-            if type(token) == bytes:
+            if isinstance(token, bytes):
                 token = token.decode('UTF-8')
             headers = {'Authorization': 'token %s' % token}
             response = requests.get(f'https://{self.ghe_instance}/api/v3', headers=headers)

detect_secrets/plugins/high_entropy_strings.py

@@ -47,7 +47,7 @@ def __init__(self, charset, limit, exclude_lines_regex, automaton, *args, **kwar
             exclude_lines_regex=exclude_lines_regex,
             false_positive_heuristics=false_positive_heuristics,
             *args,
-            **kwargs
+            **kwargs,
         )
 
     def analyze(self, file, filename, output_raw=False, output_verified_false=False):
@@ -337,7 +337,7 @@ def __init__(self, hex_limit, exclude_lines_regex=None, automaton=None, **kwargs
             limit=hex_limit,
             exclude_lines_regex=exclude_lines_regex,
             automaton=automaton,
-            **kwargs
+            **kwargs,
         )
 
     @classproperty

detect_secrets/plugins/ibm_cloud_iam.py

@@ -42,7 +42,7 @@ def verify(self, token, *args, **kwargs):
 
 
 def verify_cloud_iam_api_key(apikey):  # pragma: no cover
-    if type(apikey) == bytes:
+    if isinstance(apikey, bytes):
         apikey = apikey.decode('UTF-8')
     headers = {
         'Content-Type': 'application/x-www-form-urlencoded',

detect_secrets/plugins/keyword.py

@@ -288,7 +288,7 @@ def __init__(self, keyword_exclude=None, exclude_lines_regex=None, automaton=Non
         super(KeywordDetector, self).__init__(
             exclude_lines_regex=exclude_lines_regex,
             false_positive_heuristics=false_positive_heuristics,
-            **kwargs
+            **kwargs,
         )
 
         self.keyword_exclude = None