From b3559b42b770311eb98dec9f0f7c5175a5c7cd69 Mon Sep 17 00:00:00 2001 From: Kevin Hock Date: Sat, 4 Apr 2020 11:01:00 -0700 Subject: [PATCH 1/6] :snake: Add version bumping script Hopefully this avoids needing fixes like ca266b14dbfb6f056f36fce7c625ff20bcc08017 Note: I excluded the script from `.pre-commit-config.yaml` to use f-strings We can remove the exclusion when we drop python3.5 support --- .pre-commit-config.yaml | 5 +- scripts/bumpity.py | 132 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 136 insertions(+), 1 deletion(-) create mode 100755 scripts/bumpity.py diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 8f28bb402..a30fb242c 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -4,20 +4,23 @@ repos: hooks: - id: check-builtin-literals args: ['--no-allow-dict-kwargs'] + exclude: bumpity.py$ - id: check-docstring-first - id: debug-statements + exclude: bumpity.py$ - id: double-quote-string-fixer - id: end-of-file-fixer - id: name-tests-test - id: flake8 args: ['--max-line-length', '100'] - exclude: ^test_data/ + exclude: ^test_data/|bumpity.py$ - id: trailing-whitespace - repo: https://github.com/asottile/reorder_python_imports rev: v1.6.1 hooks: - id: reorder-python-imports language_version: python3 + exclude: bumpity.py$ - repo: https://github.com/asottile/add-trailing-comma rev: v1.4.1 hooks: diff --git a/scripts/bumpity.py b/scripts/bumpity.py new file mode 100755 index 000000000..0e871ac7a --- /dev/null +++ b/scripts/bumpity.py @@ -0,0 +1,132 @@ +#!/usr/bin/env python3 +"""Bumps the detect-secrets version, +in both `detect_secrets/__init__.py` and `README.md`. + +Then commits. +""" +import argparse +import pathlib +import subprocess +import sys + + +PROJECT_ROOT = pathlib.Path(__file__).absolute().parent.parent +INIT_FILE_PATH = PROJECT_ROOT.joinpath('detect_secrets/__init__.py') +README_FILE_PATH = PROJECT_ROOT.joinpath('README.md') + + +def _argparse_bump_type(value): + VALID_BUMP_TYPES = ('major', 'minor', 'patch') + + if value in VALID_BUMP_TYPES: + return value + + raise argparse.ArgumentTypeError( + f"Argument {value} must be one 'major', 'minor', 'patch'.", + ) + + +def parse_args(argv): + parser = argparse.ArgumentParser( + description=__doc__, + prog='bumpity', + ) + parser.add_argument( + '--bump', + help='the bump type, specified as one of {major, minor, patch}', + metavar='{major,minor,patch}', + type=_argparse_bump_type, + ) + + return parser.parse_args(argv) + + +def get_current_version(): + with open(INIT_FILE_PATH) as init_file: + first_line = init_file.read().splitlines()[0] + # e.g. VERSION = '0.13.0' + _, semver = first_line.replace(' ', '').split('=') + return map( + int, + # e.g. '0.13.0' + semver.strip("'").split('.'), + ) + + +def update_init_file(new_version): + with open(INIT_FILE_PATH, 'w') as init_file: + init_file.write(f"VERSION = '{new_version}'\n") + + +def update_readme(old_version, new_version): + with open(README_FILE_PATH, 'r') as readme: + original_text = readme.read() + with open(README_FILE_PATH, 'w') as readme: + readme.write( + original_text.replace(old_version, new_version), + ) + + +def stage_and_commit(new_version): + # Stage files + subprocess.check_output( + ( + 'git', + 'add', + INIT_FILE_PATH, + README_FILE_PATH, + ), + ) + + # Check they are the only ones staged + staged_files = subprocess.check_output( + ( + 'git', + 'diff', + '--staged', + '--name-only', + ), + ).splitlines() + if len(staged_files) != 2: + raise RuntimeWarning('More files staged than __init__.py and README.md') + + # Make the commit + subprocess.check_output( + ( + 'git', + 'commit', + '--message', + f':fist: Bumping version to {new_version}', + INIT_FILE_PATH, + README_FILE_PATH, + ), + ) + + +def main(argv=sys.argv[1:]): + if not argv: + argv.append('--help') + args = parse_args(argv) + + major, minor, patch = get_current_version() + old_version = f'{major}.{minor}.{patch}' + + if args.bump == 'major': + major += 1 + minor = 0 + patch = 0 + elif args.bump == 'minor': + minor += 1 + patch = 0 + else: + patch += 1 + + new_version = f'{major}.{minor}.{patch}' + update_init_file(new_version) + update_readme(old_version, new_version) + stage_and_commit(new_version) + print("Don't forget to update CHANGELOG.md too!") + + +if __name__ == '__main__': + sys.exit(main()) From 0b5574adb2030112ca3b01c751ace3c987a48a2b Mon Sep 17 00:00:00 2001 From: Kevin Hock Date: Sat, 4 Apr 2020 11:05:00 -0700 Subject: [PATCH 2/6] :snake: `chmod +x` the scripts in scripts/ --- scripts/benchmark.py | 0 scripts/run_performance_tests.py | 0 2 files changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 scripts/benchmark.py mode change 100644 => 100755 scripts/run_performance_tests.py diff --git a/scripts/benchmark.py b/scripts/benchmark.py old mode 100644 new mode 100755 diff --git a/scripts/run_performance_tests.py b/scripts/run_performance_tests.py old mode 100644 new mode 100755 From ff6c4a57421c40feb866ae8292d28fab576b85fe Mon Sep 17 00:00:00 2001 From: Kevin Hock Date: Sat, 4 Apr 2020 11:08:23 -0700 Subject: [PATCH 3/6] :snake: Remove unnecessary `(object)` inheritance --- detect_secrets/core/bidirectional_iterator.py | 2 +- detect_secrets/core/code_snippet.py | 2 +- detect_secrets/core/potential_secret.py | 2 +- detect_secrets/core/secrets_collection.py | 2 +- detect_secrets/core/usage.py | 8 ++++---- detect_secrets/plugins/base.py | 2 +- detect_secrets/plugins/common/ini_file_parser.py | 2 +- detect_secrets/plugins/common/yaml_file_parser.py | 2 +- testing/mocks.py | 4 ++-- tests/core/audit_test.py | 12 ++++++------ tests/core/baseline_test.py | 12 ++++++------ tests/core/bidirectional_iterator_test.py | 2 +- tests/core/potential_secret_test.py | 2 +- tests/core/secrets_collection_test.py | 8 ++++---- tests/core/usage_test.py | 2 +- tests/main_test.py | 2 +- tests/plugins/artifactory_test.py | 2 +- tests/plugins/aws_key_test.py | 2 +- tests/plugins/basic_auth_test.py | 2 +- tests/plugins/cloudant_test.py | 2 +- tests/plugins/common/filters_test.py | 6 +++--- tests/plugins/common/initialize_test.py | 4 ++-- tests/plugins/common/yaml_file_parser_test.py | 2 +- tests/plugins/high_entropy_strings_test.py | 2 +- tests/plugins/ibm_cloud_iam_test.py | 2 +- tests/plugins/ibm_cos_hmac_test.py | 2 +- tests/plugins/jwt_test.py | 2 +- tests/plugins/keyword_test.py | 2 +- tests/plugins/mailchimp_key_test.py | 2 +- tests/plugins/private_key_test.py | 2 +- tests/plugins/slack_test.py | 2 +- tests/plugins/softlayer_test.py | 2 +- tests/plugins/stripe_key_test.py | 2 +- tests/plugins/twilio_test.py | 2 +- tests/pre_commit_hook_test.py | 2 +- 35 files changed, 55 insertions(+), 55 deletions(-) diff --git a/detect_secrets/core/bidirectional_iterator.py b/detect_secrets/core/bidirectional_iterator.py index b664a99cd..a9deb6a37 100644 --- a/detect_secrets/core/bidirectional_iterator.py +++ b/detect_secrets/core/bidirectional_iterator.py @@ -1,4 +1,4 @@ -class BidirectionalIterator(object): +class BidirectionalIterator: def __init__(self, collection): self.collection = collection self.index = -1 # Starts on -1, as index is increased _before_ getting result diff --git a/detect_secrets/core/code_snippet.py b/detect_secrets/core/code_snippet.py index 5b5a997da..8d8fc0acb 100644 --- a/detect_secrets/core/code_snippet.py +++ b/detect_secrets/core/code_snippet.py @@ -43,7 +43,7 @@ def get_code_snippet(self, file_lines, line_number, lines_of_context=5): ) -class CodeSnippet(object): +class CodeSnippet: def __init__(self, snippet, start_line, target_index): """ diff --git a/detect_secrets/core/potential_secret.py b/detect_secrets/core/potential_secret.py index e7e8253f5..bf597e2ba 100644 --- a/detect_secrets/core/potential_secret.py +++ b/detect_secrets/core/potential_secret.py @@ -1,7 +1,7 @@ import hashlib -class PotentialSecret(object): +class PotentialSecret: """This custom data type represents a string found, matching the plugin rules defined in SecretsCollection, that has the potential to be a secret that we actually care about. diff --git a/detect_secrets/core/secrets_collection.py b/detect_secrets/core/secrets_collection.py index 9af886cad..c2be3ae18 100644 --- a/detect_secrets/core/secrets_collection.py +++ b/detect_secrets/core/secrets_collection.py @@ -13,7 +13,7 @@ from detect_secrets.util import build_automaton -class SecretsCollection(object): +class SecretsCollection: def __init__( self, diff --git a/detect_secrets/core/usage.py b/detect_secrets/core/usage.py index d827a75ce..ee5f7bbff 100644 --- a/detect_secrets/core/usage.py +++ b/detect_secrets/core/usage.py @@ -42,7 +42,7 @@ def add_no_verify_flag(parser): ) -class ParserBuilder(object): +class ParserBuilder: def __init__(self): self.parser = argparse.ArgumentParser() @@ -133,7 +133,7 @@ def _add_no_verify_flag(self): return self -class ScanOptions(object): +class ScanOptions: def __init__(self, subparser): self.parser = subparser.add_parser( @@ -209,7 +209,7 @@ def _add_adhoc_scanning_argument(self): ) -class AuditOptions(object): +class AuditOptions: def __init__(self, subparser): self.parser = subparser.add_parser( @@ -320,7 +320,7 @@ def get_disabled_help_text(plugin): return 'Disables {}'.format(line) -class PluginOptions(object): +class PluginOptions: all_plugins = [ PluginDescriptor.from_plugin_class(plugin, name) diff --git a/detect_secrets/plugins/base.py b/detect_secrets/plugins/base.py index 8a09c6f6d..11f39a6cd 100644 --- a/detect_secrets/plugins/base.py +++ b/detect_secrets/plugins/base.py @@ -24,7 +24,7 @@ def __get__(self, cls, owner): return classmethod(self.fget).__get__(None, owner)() -class BasePlugin(object): +class BasePlugin: """ This is an abstract class to define Plugins API. diff --git a/detect_secrets/plugins/common/ini_file_parser.py b/detect_secrets/plugins/common/ini_file_parser.py index 5aaec74a6..0b87a3fd5 100644 --- a/detect_secrets/plugins/common/ini_file_parser.py +++ b/detect_secrets/plugins/common/ini_file_parser.py @@ -17,7 +17,7 @@ def append(self, lineno, line): configparser.ParsingError = EfficientParsingError -class IniFileParser(object): +class IniFileParser: _comment_regex = re.compile(r'\s*[;#]') diff --git a/detect_secrets/plugins/common/yaml_file_parser.py b/detect_secrets/plugins/common/yaml_file_parser.py index 4f9cb2f7a..65e3efc80 100644 --- a/detect_secrets/plugins/common/yaml_file_parser.py +++ b/detect_secrets/plugins/common/yaml_file_parser.py @@ -3,7 +3,7 @@ from .constants import ALLOWLIST_REGEX -class YamlFileParser(object): +class YamlFileParser: """ Yaml config files are interesting, because they don't necessarily conform to our basic regex for detecting HighEntropyStrings as strings don't diff --git a/testing/mocks.py b/testing/mocks.py index 4ef010eb0..a52a8904e 100644 --- a/testing/mocks.py +++ b/testing/mocks.py @@ -133,7 +133,7 @@ def mock_printer(obj): """ :type obj: module """ - class PrinterShim(object): + class PrinterShim: def __init__(self): self.clear() @@ -150,7 +150,7 @@ def clear(self): @contextmanager def mock_log(namespace): - class MockLogWrapper(object): + class MockLogWrapper: """This is used to check what is being logged.""" def __init__(self): diff --git a/tests/core/audit_test.py b/tests/core/audit_test.py index ccc3eb306..a13fedaa6 100644 --- a/tests/core/audit_test.py +++ b/tests/core/audit_test.py @@ -21,7 +21,7 @@ def reset_file_cache(): audit._open_file_with_cache.cache_clear() -class TestAuditBaseline(object): +class TestAuditBaseline: def test_no_baseline(self, mock_printer): with self.mock_env(baseline='') as m: @@ -310,7 +310,7 @@ def leapfrog_baseline(self): } -class TestCompareBaselines(object): +class TestCompareBaselines: def test_raises_error_if_comparing_same_file(self): with pytest.raises(audit.RedundantComparisonError): @@ -486,7 +486,7 @@ def new_baseline(self): } -class TestDetermineAuditResults(object): +class TestDetermineAuditResults: @pytest.fixture def mock_get_raw_secret_value(self): @@ -724,7 +724,7 @@ def test_print_audit_results_none( assert expected_message in mock_printer.message -class TestPrintContext(object): +class TestPrintContext: def run_logic( self, @@ -1028,7 +1028,7 @@ def test_unicode_in_output(self, mock_printer): """)[1:-1] -class TestGetUserDecision(object): +class TestGetUserDecision: @pytest.mark.parametrize( 'user_input, expected_value', @@ -1086,7 +1086,7 @@ def mock_user_input(inputs): :type inputs: list :param inputs: list of user choices """ - class InputShim(object): + class InputShim: def __init__(self): self.message = '' self.index = 0 diff --git a/tests/core/baseline_test.py b/tests/core/baseline_test.py index bf0c08d25..021bf153e 100644 --- a/tests/core/baseline_test.py +++ b/tests/core/baseline_test.py @@ -21,7 +21,7 @@ from testing.mocks import SubprocessMock -class TestInitializeBaseline(object): +class TestInitializeBaseline: def setup(self): self.plugins = ( @@ -186,7 +186,7 @@ def test_scan_all_files_with_bad_symlinks(self): assert len(results.keys()) == 0 -class TestGetSecretsNotInBaseline(object): +class TestGetSecretsNotInBaseline: def test_nothing_new(self): # We want a secret, but just a default secret (no overriding parameters) @@ -292,7 +292,7 @@ def test_rolled_creds(self): assert baseline.data == backup_baseline -class TestUpdateBaselineWithRemovedSecrets(object): +class TestUpdateBaselineWithRemovedSecrets: def test_deleted_secret(self): new_findings = secrets_collection_factory([ @@ -393,7 +393,7 @@ def test_no_baseline_modifications(self, results_dict, baseline_dict): ) -class TestMergeBaseline(object): +class TestMergeBaseline: def test_copies_is_secret_label_accurately(self): assert merge_baseline( @@ -496,7 +496,7 @@ def test_copies_is_secret_label_accurately(self): pass -class TestMergeResults(object): +class TestMergeResults: def test_new_results_has_nothing(self): old_result = { @@ -620,7 +620,7 @@ def get_secret(): } -class TestFormatBaselineForOutput(object): +class TestFormatBaselineForOutput: def test_sorts_by_line_number_then_hash(self): output_string = format_baseline_for_output({ diff --git a/tests/core/bidirectional_iterator_test.py b/tests/core/bidirectional_iterator_test.py index 04a96ad40..f05dd327c 100644 --- a/tests/core/bidirectional_iterator_test.py +++ b/tests/core/bidirectional_iterator_test.py @@ -5,7 +5,7 @@ from detect_secrets.core import bidirectional_iterator -class TestBidirectionalIterator(object): +class TestBidirectionalIterator: def test_no_input(self): iterator = bidirectional_iterator.BidirectionalIterator([]) diff --git a/tests/core/potential_secret_test.py b/tests/core/potential_secret_test.py index 7f4dff4be..fc2e39eac 100644 --- a/tests/core/potential_secret_test.py +++ b/tests/core/potential_secret_test.py @@ -6,7 +6,7 @@ from testing.factories import potential_secret_factory -class TestPotentialSecret(object): +class TestPotentialSecret: @pytest.mark.parametrize( 'a, b, is_equal', diff --git a/tests/core/secrets_collection_test.py b/tests/core/secrets_collection_test.py index 5c8860ae9..efa8a7049 100644 --- a/tests/core/secrets_collection_test.py +++ b/tests/core/secrets_collection_test.py @@ -41,7 +41,7 @@ def mock_gmtime(): yield current_time -class TestScanFile(object): +class TestScanFile: """Testing file scanning, and interactions with different plugins.""" def test_file_is_symbolic_link(self): @@ -141,7 +141,7 @@ def test_unicode_decode_error(self, mock_log): assert len(logic.data) == 0 -class TestScanDiff(object): +class TestScanDiff: def test_success(self): secrets = self.load_from_diff().format_for_baseline_output()['results'] @@ -200,7 +200,7 @@ def load_from_diff(self, existing_secrets=None, baseline_filename='', exclude_fi return collection -class TestGetSecret(object): +class TestGetSecret: """Testing retrieval of PotentialSecret from SecretsCollection""" @pytest.mark.parametrize( @@ -258,7 +258,7 @@ def _mock_secret_hash(self, secret_hash='secret_hash'): yield -class TestBaselineInputOutput(object): +class TestBaselineInputOutput: """A critical part of the SecretsCollection is the ability to write a baseline, then read from that same baseline to recreate state. This test suite checks the functions related to that ability. diff --git a/tests/core/usage_test.py b/tests/core/usage_test.py index 3732ce5f1..f7bd76da5 100644 --- a/tests/core/usage_test.py +++ b/tests/core/usage_test.py @@ -6,7 +6,7 @@ from detect_secrets.plugins.common.util import import_plugins -class TestPluginOptions(object): +class TestPluginOptions: @staticmethod def parse_args(argument_string=''): diff --git a/tests/main_test.py b/tests/main_test.py index ed682e9f5..597b3b80a 100644 --- a/tests/main_test.py +++ b/tests/main_test.py @@ -74,7 +74,7 @@ def get_plugin_report(extra=None): ) + '\n' -class TestMain(object): +class TestMain: """These are smoke tests for the console usage of detect_secrets. Most of the functional test cases should be within their own module tests. """ diff --git a/tests/plugins/artifactory_test.py b/tests/plugins/artifactory_test.py index f011343b1..79d2e1828 100644 --- a/tests/plugins/artifactory_test.py +++ b/tests/plugins/artifactory_test.py @@ -5,7 +5,7 @@ from detect_secrets.plugins.artifactory import ArtifactoryDetector -class TestArtifactoryDetector(object): +class TestArtifactoryDetector: @pytest.mark.parametrize( 'payload, should_flag', diff --git a/tests/plugins/aws_key_test.py b/tests/plugins/aws_key_test.py index 10a481f0d..a5d430429 100644 --- a/tests/plugins/aws_key_test.py +++ b/tests/plugins/aws_key_test.py @@ -15,7 +15,7 @@ EXAMPLE_SECRET = 'wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY' -class TestAWSKeyDetector(object): +class TestAWSKeyDetector: def setup(self): self.example_key = 'AKIAZZZZZZZZZZZZZZZZ' diff --git a/tests/plugins/basic_auth_test.py b/tests/plugins/basic_auth_test.py index 9c4f41f04..6b3e0e2c7 100644 --- a/tests/plugins/basic_auth_test.py +++ b/tests/plugins/basic_auth_test.py @@ -5,7 +5,7 @@ from detect_secrets.plugins.basic_auth import BasicAuthDetector -class TestBasicAuthDetector(object): +class TestBasicAuthDetector: @pytest.mark.parametrize( 'payload, should_flag', diff --git a/tests/plugins/cloudant_test.py b/tests/plugins/cloudant_test.py index 14de5a741..bb5424d4c 100644 --- a/tests/plugins/cloudant_test.py +++ b/tests/plugins/cloudant_test.py @@ -17,7 +17,7 @@ CL_API_KEY = 'abcdefghijabcdefghijabcd' -class TestCloudantDetector(object): +class TestCloudantDetector: @pytest.mark.parametrize( 'payload, should_flag', diff --git a/tests/plugins/common/filters_test.py b/tests/plugins/common/filters_test.py index 11b1236d2..0d9dbd150 100644 --- a/tests/plugins/common/filters_test.py +++ b/tests/plugins/common/filters_test.py @@ -5,7 +5,7 @@ from detect_secrets.plugins.common import filters -class TestIsSequentialString(object): +class TestIsSequentialString: @pytest.mark.parametrize( 'secret', ( @@ -43,7 +43,7 @@ def test_failure(self, secret): assert not filters.is_sequential_string(secret) -class TestIsLikelyIdString(object): +class TestIsLikelyIdString: @pytest.mark.parametrize( 'secret, line', [ @@ -75,7 +75,7 @@ def test_failure(self, secret, line): assert not filters.is_likely_id_string(secret, line) -class TestIsPotentialUuid(object): +class TestIsPotentialUuid: @pytest.mark.parametrize( 'secret', [ diff --git a/tests/plugins/common/initialize_test.py b/tests/plugins/common/initialize_test.py index ffdf80bfd..357161763 100644 --- a/tests/plugins/common/initialize_test.py +++ b/tests/plugins/common/initialize_test.py @@ -8,7 +8,7 @@ from detect_secrets.plugins.high_entropy_strings import HexHighEntropyString -class TestFromPluginClassname(object): +class TestFromPluginClassname: def test_success(self): plugin = initialize.from_plugin_classname( @@ -39,7 +39,7 @@ def test_fails_on_bad_initialization(self): ) -class TestFromSecretType(object): +class TestFromSecretType: def setup(self): self.settings = [ diff --git a/tests/plugins/common/yaml_file_parser_test.py b/tests/plugins/common/yaml_file_parser_test.py index d1972dd55..81f112dd9 100644 --- a/tests/plugins/common/yaml_file_parser_test.py +++ b/tests/plugins/common/yaml_file_parser_test.py @@ -8,7 +8,7 @@ from testing.mocks import mock_file_object -class TestYamlFileParser(object): +class TestYamlFileParser: def test_get_ignored_lines(self): content = """keyA: value diff --git a/tests/plugins/high_entropy_strings_test.py b/tests/plugins/high_entropy_strings_test.py index 2dec6c79a..87fde5146 100644 --- a/tests/plugins/high_entropy_strings_test.py +++ b/tests/plugins/high_entropy_strings_test.py @@ -10,7 +10,7 @@ from testing.mocks import mock_file_object -class HighEntropyStringsTest(object): +class HighEntropyStringsTest: """ Some explaining should be done regarding the "enforced" format of the parametrized abstract pytests. diff --git a/tests/plugins/ibm_cloud_iam_test.py b/tests/plugins/ibm_cloud_iam_test.py index b42f28ea3..ff972d7b7 100644 --- a/tests/plugins/ibm_cloud_iam_test.py +++ b/tests/plugins/ibm_cloud_iam_test.py @@ -11,7 +11,7 @@ CLOUD_IAM_KEY_BYTES = b'abcd1234abcd1234abcd1234ABCD1234ABCD1234--__' -class TestIBMCloudIamDetector(object): +class TestIBMCloudIamDetector: @pytest.mark.parametrize( 'payload, should_flag', diff --git a/tests/plugins/ibm_cos_hmac_test.py b/tests/plugins/ibm_cos_hmac_test.py index b4d05b273..bb218d84d 100644 --- a/tests/plugins/ibm_cos_hmac_test.py +++ b/tests/plugins/ibm_cos_hmac_test.py @@ -17,7 +17,7 @@ SECRET_ACCESS_KEY = '1234567890abcdef1234567890abcdef1234567890abcdef' -class TestIbmCosHmacDetector(object): +class TestIbmCosHmacDetector: @pytest.mark.parametrize( 'payload, should_flag', diff --git a/tests/plugins/jwt_test.py b/tests/plugins/jwt_test.py index 90bba26b2..070b7df63 100644 --- a/tests/plugins/jwt_test.py +++ b/tests/plugins/jwt_test.py @@ -5,7 +5,7 @@ from detect_secrets.plugins.jwt import JwtTokenDetector -class TestJwtTokenDetector(object): +class TestJwtTokenDetector: @pytest.mark.parametrize( 'payload, should_flag', diff --git a/tests/plugins/keyword_test.py b/tests/plugins/keyword_test.py index 8e658e300..2de77de8c 100644 --- a/tests/plugins/keyword_test.py +++ b/tests/plugins/keyword_test.py @@ -175,7 +175,7 @@ ) -class TestKeywordDetector(object): +class TestKeywordDetector: @pytest.mark.parametrize( 'file_content', diff --git a/tests/plugins/mailchimp_key_test.py b/tests/plugins/mailchimp_key_test.py index b16cab1b1..e74d259fa 100644 --- a/tests/plugins/mailchimp_key_test.py +++ b/tests/plugins/mailchimp_key_test.py @@ -7,7 +7,7 @@ from testing.mocks import mock_file_object -class TestMailchimpKeyDetector(object): +class TestMailchimpKeyDetector: @pytest.mark.parametrize( 'file_content,should_flag', diff --git a/tests/plugins/private_key_test.py b/tests/plugins/private_key_test.py index f7dca3ec4..5a037e191 100644 --- a/tests/plugins/private_key_test.py +++ b/tests/plugins/private_key_test.py @@ -7,7 +7,7 @@ from testing.mocks import mock_file_object -class TestPrivateKeyDetector(object): +class TestPrivateKeyDetector: @pytest.mark.parametrize( 'file_content', diff --git a/tests/plugins/slack_test.py b/tests/plugins/slack_test.py index d0a4e03ec..111724e66 100644 --- a/tests/plugins/slack_test.py +++ b/tests/plugins/slack_test.py @@ -7,7 +7,7 @@ from testing.mocks import mock_file_object -class TestSlackDetector(object): +class TestSlackDetector: @pytest.mark.parametrize( 'file_content', diff --git a/tests/plugins/softlayer_test.py b/tests/plugins/softlayer_test.py index 5cf88fb01..a21f0df73 100644 --- a/tests/plugins/softlayer_test.py +++ b/tests/plugins/softlayer_test.py @@ -13,7 +13,7 @@ SL_TOKEN = 'abcd1234abcd1234abcd1234abcd1234abcd1234abcd1234abcd1234abcd1234' -class TestSoftlayerDetector(object): +class TestSoftlayerDetector: @pytest.mark.parametrize( 'payload, should_flag', diff --git a/tests/plugins/stripe_key_test.py b/tests/plugins/stripe_key_test.py index 328b51e90..529a086c3 100644 --- a/tests/plugins/stripe_key_test.py +++ b/tests/plugins/stripe_key_test.py @@ -7,7 +7,7 @@ from testing.mocks import mock_file_object -class TestStripeKeyDetector(object): +class TestStripeKeyDetector: @pytest.mark.parametrize( 'file_content,should_flag', diff --git a/tests/plugins/twilio_test.py b/tests/plugins/twilio_test.py index 3d2c83fd9..58001971a 100644 --- a/tests/plugins/twilio_test.py +++ b/tests/plugins/twilio_test.py @@ -5,7 +5,7 @@ from detect_secrets.plugins.twilio import TwilioKeyDetector -class TestTwilioKeyDetector(object): +class TestTwilioKeyDetector: @pytest.mark.parametrize( 'payload, should_flag', diff --git a/tests/pre_commit_hook_test.py b/tests/pre_commit_hook_test.py index b112e7f3c..43542d2de 100644 --- a/tests/pre_commit_hook_test.py +++ b/tests/pre_commit_hook_test.py @@ -28,7 +28,7 @@ def assert_commit_succeeds(command): assert pre_commit_hook.main(command.split()) == 0 -class TestPreCommitHook(object): +class TestPreCommitHook: def test_file_with_secrets(self, mock_log): assert_commit_blocked('test_data/files/file_with_secrets.py') From eac69193dbbd72320e0e07edd6db289f965cd651 Mon Sep 17 00:00:00 2001 From: Kevin Hock Date: Sat, 4 Apr 2020 11:10:32 -0700 Subject: [PATCH 4/6] :bug: Catch TypeError from `from_plugin_classname` This fixes issue #269. Previously we showed an unhelpful stack trace. --- detect_secrets/plugins/common/initialize.py | 7 ++++++- detect_secrets/pre_commit_hook.py | 2 +- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/detect_secrets/plugins/common/initialize.py b/detect_secrets/plugins/common/initialize.py index 3ad337caf..7df4553b3 100644 --- a/detect_secrets/plugins/common/initialize.py +++ b/detect_secrets/plugins/common/initialize.py @@ -165,7 +165,12 @@ def from_plugin_classname( try: klass = import_plugins()[plugin_classname] except KeyError: - log.warning('No such plugin to initialize.') + log.error('Error: No such `{}` plugin to initialize.'.format(plugin_classname)) + log.error('Chances are you should run `pre-commit autoupdate`.') + log.error( + 'This error occurs when using a baseline that was made by ' + 'a newer detect-secrets version than the one running.', + ) raise TypeError try: diff --git a/detect_secrets/pre_commit_hook.py b/detect_secrets/pre_commit_hook.py index b6740d77b..b516e4822 100644 --- a/detect_secrets/pre_commit_hook.py +++ b/detect_secrets/pre_commit_hook.py @@ -32,7 +32,7 @@ def main(argv=None): # If baseline is provided, we first want to make sure # it's valid, before doing any further computation. baseline_collection = get_baseline(args.baseline[0]) - except (IOError, ValueError): + except (IOError, TypeError, ValueError): # Error logs handled within logic. return 1 From 5ca1d7ea770a9e79d4d9ecaa5ed2f519524c216f Mon Sep 17 00:00:00 2001 From: Kevin Hock Date: Sat, 4 Apr 2020 11:11:54 -0700 Subject: [PATCH 5/6] :snake: Remove all __future__ imports Missed a few in 5d0d4612d5848a8bd71c8686a20ddf0605f9bf40 Ran `grep -rnw --include=*.py -e 'future' | grep -v three_six | grep -v .tox` this time. --- scripts/benchmark.py | 2 -- scripts/run_performance_tests.py | 3 --- testing/factories.py | 2 -- tests/core/audit_test.py | 3 --- tests/core/baseline_test.py | 2 -- tests/core/bidirectional_iterator_test.py | 2 -- tests/core/potential_secret_test.py | 3 --- tests/core/secrets_collection_test.py | 2 -- tests/core/usage_test.py | 2 -- tests/plugins/artifactory_test.py | 2 -- tests/plugins/aws_key_test.py | 3 --- tests/plugins/base_test.py | 3 --- tests/plugins/basic_auth_test.py | 2 -- tests/plugins/cloudant_test.py | 2 -- tests/plugins/common/filters_test.py | 2 -- tests/plugins/common/initialize_test.py | 2 -- tests/plugins/common/yaml_file_parser_test.py | 3 --- tests/plugins/high_entropy_strings_test.py | 3 --- tests/plugins/ibm_cloud_iam_test.py | 2 -- tests/plugins/ibm_cos_hmac_test.py | 2 -- tests/plugins/jwt_test.py | 2 -- tests/plugins/keyword_test.py | 3 --- tests/plugins/mailchimp_key_test.py | 3 --- tests/plugins/private_key_test.py | 3 --- tests/plugins/slack_test.py | 3 --- tests/plugins/softlayer_test.py | 2 -- tests/plugins/stripe_key_test.py | 3 --- tests/plugins/twilio_test.py | 2 -- tests/pre_commit_hook_test.py | 2 -- 29 files changed, 70 deletions(-) diff --git a/scripts/benchmark.py b/scripts/benchmark.py index 66f5c21f5..30f62fd6e 100755 --- a/scripts/benchmark.py +++ b/scripts/benchmark.py @@ -1,6 +1,4 @@ #!/usr/bin/python3 -from __future__ import print_function - import argparse import json import os diff --git a/scripts/run_performance_tests.py b/scripts/run_performance_tests.py index 28042488c..bbcaba184 100755 --- a/scripts/run_performance_tests.py +++ b/scripts/run_performance_tests.py @@ -1,6 +1,3 @@ -#!/usr/bin/env python3 -from __future__ import print_function - import argparse import json import os diff --git a/testing/factories.py b/testing/factories.py index db58cee6a..b89c123d9 100644 --- a/testing/factories.py +++ b/testing/factories.py @@ -1,5 +1,3 @@ -from __future__ import absolute_import - from detect_secrets.core.potential_secret import PotentialSecret from detect_secrets.core.secrets_collection import SecretsCollection diff --git a/tests/core/audit_test.py b/tests/core/audit_test.py index a13fedaa6..024554457 100644 --- a/tests/core/audit_test.py +++ b/tests/core/audit_test.py @@ -1,6 +1,3 @@ -from __future__ import absolute_import -from __future__ import unicode_literals - import string import textwrap from contextlib import contextmanager diff --git a/tests/core/baseline_test.py b/tests/core/baseline_test.py index 021bf153e..360a39d4e 100644 --- a/tests/core/baseline_test.py +++ b/tests/core/baseline_test.py @@ -1,5 +1,3 @@ -from __future__ import absolute_import - import json import random diff --git a/tests/core/bidirectional_iterator_test.py b/tests/core/bidirectional_iterator_test.py index f05dd327c..08fe64a84 100644 --- a/tests/core/bidirectional_iterator_test.py +++ b/tests/core/bidirectional_iterator_test.py @@ -1,5 +1,3 @@ -from __future__ import absolute_import - import pytest from detect_secrets.core import bidirectional_iterator diff --git a/tests/core/potential_secret_test.py b/tests/core/potential_secret_test.py index fc2e39eac..db10cdf28 100644 --- a/tests/core/potential_secret_test.py +++ b/tests/core/potential_secret_test.py @@ -1,6 +1,3 @@ -#!/usr/bin/python -from __future__ import absolute_import - import pytest from testing.factories import potential_secret_factory diff --git a/tests/core/secrets_collection_test.py b/tests/core/secrets_collection_test.py index efa8a7049..0b7deed08 100644 --- a/tests/core/secrets_collection_test.py +++ b/tests/core/secrets_collection_test.py @@ -1,5 +1,3 @@ -from __future__ import absolute_import - import hashlib import json from contextlib import contextmanager diff --git a/tests/core/usage_test.py b/tests/core/usage_test.py index f7bd76da5..6cf3ee40f 100644 --- a/tests/core/usage_test.py +++ b/tests/core/usage_test.py @@ -1,5 +1,3 @@ -from __future__ import absolute_import - import pytest from detect_secrets.core.usage import ParserBuilder diff --git a/tests/plugins/artifactory_test.py b/tests/plugins/artifactory_test.py index 79d2e1828..d13c08dd9 100644 --- a/tests/plugins/artifactory_test.py +++ b/tests/plugins/artifactory_test.py @@ -1,5 +1,3 @@ -from __future__ import absolute_import - import pytest from detect_secrets.plugins.artifactory import ArtifactoryDetector diff --git a/tests/plugins/aws_key_test.py b/tests/plugins/aws_key_test.py index a5d430429..f47d40e95 100644 --- a/tests/plugins/aws_key_test.py +++ b/tests/plugins/aws_key_test.py @@ -1,6 +1,3 @@ -from __future__ import absolute_import -from __future__ import unicode_literals - import textwrap import mock diff --git a/tests/plugins/base_test.py b/tests/plugins/base_test.py index 2dd77cad0..1c332e30c 100644 --- a/tests/plugins/base_test.py +++ b/tests/plugins/base_test.py @@ -1,6 +1,3 @@ -from __future__ import absolute_import -from __future__ import unicode_literals - from contextlib import contextmanager import mock diff --git a/tests/plugins/basic_auth_test.py b/tests/plugins/basic_auth_test.py index 6b3e0e2c7..9e7d37fd5 100644 --- a/tests/plugins/basic_auth_test.py +++ b/tests/plugins/basic_auth_test.py @@ -1,5 +1,3 @@ -from __future__ import absolute_import - import pytest from detect_secrets.plugins.basic_auth import BasicAuthDetector diff --git a/tests/plugins/cloudant_test.py b/tests/plugins/cloudant_test.py index bb5424d4c..68da513af 100644 --- a/tests/plugins/cloudant_test.py +++ b/tests/plugins/cloudant_test.py @@ -1,5 +1,3 @@ -from __future__ import absolute_import - import textwrap import pytest diff --git a/tests/plugins/common/filters_test.py b/tests/plugins/common/filters_test.py index 0d9dbd150..f341f9354 100644 --- a/tests/plugins/common/filters_test.py +++ b/tests/plugins/common/filters_test.py @@ -1,5 +1,3 @@ -from __future__ import absolute_import - import pytest from detect_secrets.plugins.common import filters diff --git a/tests/plugins/common/initialize_test.py b/tests/plugins/common/initialize_test.py index 357161763..c3aa26a41 100644 --- a/tests/plugins/common/initialize_test.py +++ b/tests/plugins/common/initialize_test.py @@ -1,5 +1,3 @@ -from __future__ import absolute_import - import mock import pytest diff --git a/tests/plugins/common/yaml_file_parser_test.py b/tests/plugins/common/yaml_file_parser_test.py index 81f112dd9..8e718f636 100644 --- a/tests/plugins/common/yaml_file_parser_test.py +++ b/tests/plugins/common/yaml_file_parser_test.py @@ -1,6 +1,3 @@ -from __future__ import absolute_import -from __future__ import unicode_literals - import mock import pytest diff --git a/tests/plugins/high_entropy_strings_test.py b/tests/plugins/high_entropy_strings_test.py index 87fde5146..29bc99321 100644 --- a/tests/plugins/high_entropy_strings_test.py +++ b/tests/plugins/high_entropy_strings_test.py @@ -1,6 +1,3 @@ -from __future__ import absolute_import -from __future__ import unicode_literals - import codecs import pytest diff --git a/tests/plugins/ibm_cloud_iam_test.py b/tests/plugins/ibm_cloud_iam_test.py index ff972d7b7..75e6d6e5a 100644 --- a/tests/plugins/ibm_cloud_iam_test.py +++ b/tests/plugins/ibm_cloud_iam_test.py @@ -1,5 +1,3 @@ -from __future__ import absolute_import - import pytest import responses diff --git a/tests/plugins/ibm_cos_hmac_test.py b/tests/plugins/ibm_cos_hmac_test.py index bb218d84d..432b763eb 100644 --- a/tests/plugins/ibm_cos_hmac_test.py +++ b/tests/plugins/ibm_cos_hmac_test.py @@ -1,5 +1,3 @@ -from __future__ import absolute_import - import textwrap import pytest diff --git a/tests/plugins/jwt_test.py b/tests/plugins/jwt_test.py index 070b7df63..bff064577 100644 --- a/tests/plugins/jwt_test.py +++ b/tests/plugins/jwt_test.py @@ -1,5 +1,3 @@ -from __future__ import absolute_import - import pytest from detect_secrets.plugins.jwt import JwtTokenDetector diff --git a/tests/plugins/keyword_test.py b/tests/plugins/keyword_test.py index 2de77de8c..35fcf86fb 100644 --- a/tests/plugins/keyword_test.py +++ b/tests/plugins/keyword_test.py @@ -1,6 +1,3 @@ -from __future__ import absolute_import -from __future__ import unicode_literals - import ahocorasick import pytest diff --git a/tests/plugins/mailchimp_key_test.py b/tests/plugins/mailchimp_key_test.py index e74d259fa..4820dc9ca 100644 --- a/tests/plugins/mailchimp_key_test.py +++ b/tests/plugins/mailchimp_key_test.py @@ -1,6 +1,3 @@ -from __future__ import absolute_import -from __future__ import unicode_literals - import pytest from detect_secrets.plugins.mailchimp import MailchimpDetector diff --git a/tests/plugins/private_key_test.py b/tests/plugins/private_key_test.py index 5a037e191..288326689 100644 --- a/tests/plugins/private_key_test.py +++ b/tests/plugins/private_key_test.py @@ -1,6 +1,3 @@ -from __future__ import absolute_import -from __future__ import unicode_literals - import pytest from detect_secrets.plugins.private_key import PrivateKeyDetector diff --git a/tests/plugins/slack_test.py b/tests/plugins/slack_test.py index 111724e66..af945c092 100644 --- a/tests/plugins/slack_test.py +++ b/tests/plugins/slack_test.py @@ -1,6 +1,3 @@ -from __future__ import absolute_import -from __future__ import unicode_literals - import pytest from detect_secrets.plugins.slack import SlackDetector diff --git a/tests/plugins/softlayer_test.py b/tests/plugins/softlayer_test.py index a21f0df73..59aca10ee 100644 --- a/tests/plugins/softlayer_test.py +++ b/tests/plugins/softlayer_test.py @@ -1,5 +1,3 @@ -from __future__ import absolute_import - import textwrap import pytest diff --git a/tests/plugins/stripe_key_test.py b/tests/plugins/stripe_key_test.py index 529a086c3..4552a3dfd 100644 --- a/tests/plugins/stripe_key_test.py +++ b/tests/plugins/stripe_key_test.py @@ -1,6 +1,3 @@ -from __future__ import absolute_import -from __future__ import unicode_literals - import pytest from detect_secrets.plugins.stripe import StripeDetector diff --git a/tests/plugins/twilio_test.py b/tests/plugins/twilio_test.py index 58001971a..83f9ca3d6 100644 --- a/tests/plugins/twilio_test.py +++ b/tests/plugins/twilio_test.py @@ -1,5 +1,3 @@ -from __future__ import absolute_import - import pytest from detect_secrets.plugins.twilio import TwilioKeyDetector diff --git a/tests/pre_commit_hook_test.py b/tests/pre_commit_hook_test.py index 43542d2de..aa4941d58 100644 --- a/tests/pre_commit_hook_test.py +++ b/tests/pre_commit_hook_test.py @@ -1,5 +1,3 @@ -from __future__ import absolute_import - import json from contextlib import contextmanager From 74d37877239da0cc16c9b35cdd266c4875c5b3c5 Mon Sep 17 00:00:00 2001 From: Kevin Hock Date: Mon, 6 Apr 2020 22:31:53 -0700 Subject: [PATCH 6/6] :performing_arts: Improve the performance of line regexes This fixes issue #244. Only check the line for allowlist regexes or --exclude-lines if a secret was found. --- detect_secrets/core/secrets_collection.py | 3 ++ detect_secrets/plugins/base.py | 46 ++++++++++++---------- tests/plugins/high_entropy_strings_test.py | 21 +++++++--- 3 files changed, 45 insertions(+), 25 deletions(-) diff --git a/detect_secrets/core/secrets_collection.py b/detect_secrets/core/secrets_collection.py index c2be3ae18..2ba5f8bf0 100644 --- a/detect_secrets/core/secrets_collection.py +++ b/detect_secrets/core/secrets_collection.py @@ -156,6 +156,8 @@ def scan_diff( at incremental differences, rather than re-scanning the codebase every time. This function supports this, and adds information to self.data. + Note that this is only called by detect-secrets-server. + :type diff: str :param diff: diff string. e.g. The output of `git diff ` @@ -338,6 +340,7 @@ def _extract_secrets_from_patch(self, f, plugin, filename): """Extract secrets from a given patch file object. Note that we only want to capture incoming secrets (so added lines). + Note that this is only called by detect-secrets-server. :type f: unidiff.patch.PatchedFile :type plugin: detect_secrets.plugins.base.BasePlugin diff --git a/detect_secrets/plugins/base.py b/detect_secrets/plugins/base.py index 11f39a6cd..b8ea2f74d 100644 --- a/detect_secrets/plugins/base.py +++ b/detect_secrets/plugins/base.py @@ -70,17 +70,15 @@ def __init__( :param false_positive_heuristics: List of fp-heuristic functions applicable to this plugin """ - self.exclude_lines_regex = None - if exclude_lines_regex: - self.exclude_lines_regex = re.compile(exclude_lines_regex) + self.exclude_lines_regex = ( + re.compile(exclude_lines_regex) + if exclude_lines_regex + else None + ) self.should_verify = should_verify - self.false_positive_heuristics = ( - false_positive_heuristics - if false_positive_heuristics - else [] - ) + self.false_positive_heuristics = false_positive_heuristics or [] @classproperty def disable_flag_text(cls): @@ -101,6 +99,19 @@ def disable_flag_text(cls): def default_options(cls): return {} + def _is_excluded_line(self, line): + return ( + any( + allowlist_regex.search(line) + for allowlist_regex in ALLOWLIST_REGEXES + ) + or + ( + self.exclude_lines_regex and + self.exclude_lines_regex.search(line) + ) + ) + def analyze(self, file, filename): """ :param file: The File object itself. @@ -114,6 +125,13 @@ def analyze(self, file, filename): file_lines = tuple(file.readlines()) for line_num, line in enumerate(file_lines, start=1): results = self.analyze_line(line, line_num, filename) + if ( + not results + or + self._is_excluded_line(line) + ): + continue + if not self.should_verify: potential_secrets.update(results) continue @@ -146,18 +164,6 @@ def analyze_line(self, string, line_num, filename): NOTE: line_num and filename are used for PotentialSecret creation only. """ - if ( - any( - allowlist_regex.search(string) for allowlist_regex in ALLOWLIST_REGEXES - ) - - or ( - self.exclude_lines_regex and - self.exclude_lines_regex.search(string) - ) - ): - return {} - return self.analyze_string_content( string, line_num, diff --git a/tests/plugins/high_entropy_strings_test.py b/tests/plugins/high_entropy_strings_test.py index 29bc99321..450dd340d 100644 --- a/tests/plugins/high_entropy_strings_test.py +++ b/tests/plugins/high_entropy_strings_test.py @@ -297,18 +297,29 @@ def test_discounts_when_all_numbers(self): ) # This makes sure discounting works. - assert self.logic.calculate_shannon_entropy('0123456789') < \ + assert ( + self.logic.calculate_shannon_entropy('0123456789') + < original_scanner.calculate_shannon_entropy('0123456789') - + ) # This is the goal. assert self.logic.calculate_shannon_entropy('0123456789') < 3 # This makes sure it is length dependent. - assert self.logic.calculate_shannon_entropy('0123456789') < \ + assert ( + self.logic.calculate_shannon_entropy('0123456789') + < self.logic.calculate_shannon_entropy('01234567890123456789') + ) # This makes sure it only occurs with numbers. - assert self.logic.calculate_shannon_entropy('12345a') == \ + assert ( + self.logic.calculate_shannon_entropy('12345a') + == original_scanner.calculate_shannon_entropy('12345a') - assert self.logic.calculate_shannon_entropy('0') == \ + ) + assert ( + self.logic.calculate_shannon_entropy('0') + == original_scanner.calculate_shannon_entropy('0') + )