Thanks to visit codestin.com
Credit goes to github.com

Skip to content

Commit 3783eaf

Browse files
author
Aaron Loo
committed
--audit v1
1 parent f5de147 commit 3783eaf

12 files changed

Lines changed: 565 additions & 8 deletions

File tree

‎detect_secrets/core/audit.py‎

Lines changed: 150 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,150 @@
1+
from __future__ import print_function
2+
3+
import json
4+
import subprocess
5+
import sys
6+
from builtins import input
7+
from collections import defaultdict
8+
9+
from .baseline import merge_results
10+
from .color import BashColor
11+
from .color import Color
12+
13+
14+
def audit_baseline(baseline_filename):
15+
original_baseline = _get_baseline_from_file(baseline_filename)
16+
if not original_baseline:
17+
return
18+
19+
results = defaultdict(list)
20+
for filename, secret in _secret_generator(original_baseline):
21+
_clear_screen()
22+
_print_context(filename, secret)
23+
24+
decision = _get_user_decision()
25+
if decision == 'q':
26+
print('Quitting...')
27+
break
28+
29+
_handle_user_decision(decision, secret)
30+
results[filename].append(secret)
31+
32+
print('Saving progress...')
33+
original_baseline['results'] = merge_results(
34+
original_baseline['results'],
35+
dict(results),
36+
)
37+
_save_baseline_to_file(baseline_filename, original_baseline)
38+
39+
40+
def _clear_screen():
41+
subprocess.call(['clear'])
42+
43+
44+
def _print_context(filename, secret): # pragma: no cover
45+
print('{} {}'.format(
46+
BashColor.color(
47+
'Filename:',
48+
Color.BOLD,
49+
),
50+
BashColor.color(
51+
filename,
52+
Color.PURPLE,
53+
),
54+
))
55+
print('-' * 10)
56+
print(_get_secret_with_context(
57+
filename,
58+
secret['line_number'],
59+
5,
60+
))
61+
print('-' * 10)
62+
63+
64+
def _handle_user_decision(decision, secret):
65+
if decision == 'y':
66+
secret['is_secret'] = True
67+
elif decision == 'n':
68+
secret['is_secret'] = False
69+
70+
71+
def _get_baseline_from_file(filename): # pragma: no cover
72+
try:
73+
with open(filename) as f:
74+
return json.loads(f.read())
75+
except (IOError, json.decoder.JSONDecodeError):
76+
print('Not a valid baseline file!', file=sys.stderr)
77+
return
78+
79+
80+
def _save_baseline_to_file(filename, data):
81+
with open(filename, 'w') as f:
82+
f.write(json.dumps(
83+
data,
84+
indent=2,
85+
sort_keys=True,
86+
))
87+
88+
89+
def _secret_generator(baseline):
90+
for filename, secrets in baseline['results'].items():
91+
for secret in secrets:
92+
try:
93+
secret['is_secret']
94+
except KeyError:
95+
yield filename, secret
96+
97+
break
98+
99+
100+
def _get_secret_with_context(filename, secret_lineno, lines_of_context):
101+
"""
102+
Displays the secret, with surrounding lines of code for better context.
103+
104+
:type filename: str
105+
:param filename: filename where secret resides in
106+
107+
:type secret_lineno: int
108+
:param secret_lineno: line where secret is found
109+
110+
:type lines_of_context: int
111+
:param lines_of_context: number of lines displayed before and after
112+
secret.
113+
"""
114+
start_line = 1 if secret_lineno <= lines_of_context \
115+
else secret_lineno - lines_of_context
116+
end_line = secret_lineno + lines_of_context
117+
118+
output = subprocess.check_output([
119+
'sed',
120+
'-n', '{},{}p'.format(start_line, end_line),
121+
filename,
122+
]).decode('utf-8')
123+
124+
# TODO: Highlight the secret found.
125+
126+
return '\n'.join(
127+
map(
128+
lambda x: '{}:{}'.format(
129+
BashColor.color(
130+
str(int(x[0]) + start_line),
131+
Color.LIGHT_GREEN,
132+
),
133+
x[1]
134+
),
135+
enumerate(output.splitlines()),
136+
)
137+
)
138+
139+
140+
def _get_user_decision():
141+
user_input = None
142+
while user_input not in ['y', 'n', 's', 'q']:
143+
if user_input:
144+
print('Invalid input.')
145+
146+
user_input = input('Is this a valid secret? (y)es, (n)o, (s)kip, (q)uit: ')
147+
if user_input:
148+
user_input = user_input[0].lower()
149+
150+
return user_input

‎detect_secrets/core/baseline.py‎

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -146,6 +146,36 @@ def update_baseline_with_removed_secrets(results, baseline, filelist):
146146
return updated
147147

148148

149+
def merge_results(old_results, new_results):
150+
"""Update results in baseline with latest information.
151+
152+
As a rule of thumb, we want to favor the new results, yet at the same
153+
time, transfer non-modified data from the old results set.
154+
155+
Assumptions:
156+
* The list of results in each secret set is in the same order.
157+
This means that new_results cannot have *more* results than
158+
old_results.
159+
160+
:type old_results: dict
161+
:param old_results: results of status quo
162+
163+
:type new_results: dict
164+
:param new_results: results to replace status quo
165+
166+
:rtype: dict
167+
"""
168+
for filename, secrets in old_results.items():
169+
if filename not in new_results:
170+
new_results[filename] = secrets
171+
break
172+
173+
if len(secrets) > len(new_results[filename]):
174+
new_results[filename].extend(secrets[len(new_results[filename]):])
175+
176+
return new_results
177+
178+
149179
def _get_git_tracked_files(rootdir='.'):
150180
"""Parsing .gitignore rules is hard.
151181

‎detect_secrets/core/color.py‎

Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,38 @@
1+
from enum import Enum
2+
3+
4+
class Color(Enum):
5+
NORMAL = 0
6+
BOLD = 1
7+
8+
RED = 2
9+
LIGHT_GREEN = 3
10+
PURPLE = 4
11+
12+
13+
class _BashColor(object):
14+
15+
PREFIX = '\033'
16+
17+
def color(self, text, color):
18+
"""
19+
:type text: str
20+
:param text: the text to colorize
21+
22+
:type color: Color
23+
:param color: the color to make the text
24+
25+
:returns: colored string
26+
"""
27+
color_map = {
28+
Color.BOLD: '[1m',
29+
Color.RED: '[41m',
30+
Color.LIGHT_GREEN: '[92m',
31+
Color.PURPLE: '[95m',
32+
}
33+
34+
return self.PREFIX + color_map[color] + text + \
35+
self.PREFIX + '[0m'
36+
37+
38+
BashColor = _BashColor()

‎detect_secrets/core/usage.py‎

Lines changed: 15 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,8 @@ def add_pre_commit_arguments(self):
2121
._add_set_baseline_argument()
2222

2323
def add_console_use_arguments(self):
24-
return self._add_initialize_baseline_argument()
24+
return self._add_initialize_baseline_argument()\
25+
._add_audit_baseline_argument()
2526

2627
def parse_args(self, argv):
2728
output = self.parser.parse_args(argv)
@@ -73,6 +74,19 @@ def _add_initialize_baseline_argument(self):
7374

7475
return self
7576

77+
def _add_audit_baseline_argument(self):
78+
self.parser.add_argument(
79+
'--audit',
80+
nargs=1,
81+
metavar='BASELINE_FILE_TO_AUDIT',
82+
help=(
83+
'Audit a given baseline file to distinguish the difference '
84+
'between false and true positives.'
85+
),
86+
)
87+
88+
return self
89+
7690

7791
class PluginDescriptor(namedtuple(
7892
'PluginDescriptor',

‎detect_secrets/main.py‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@
55
import json
66
import sys
77

8+
from detect_secrets.core import audit
89
from detect_secrets.core import baseline
910
from detect_secrets.core.log import CustomLog
1011
from detect_secrets.core.usage import ParserBuilder
@@ -41,6 +42,8 @@ def main(argv=None):
4142
sort_keys=True
4243
)
4344
)
45+
elif args.audit:
46+
audit.audit_baseline(args.audit[0])
4447

4548
return 0
4649

‎detect_secrets/plugins/base.py‎

Lines changed: 11 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,16 @@
1+
from abc import ABCMeta
2+
from abc import abstractmethod
3+
4+
15
class BasePlugin(object):
26
"""This is an abstract class to define Plugins API"""
37

8+
__metaclass__ = ABCMeta
9+
secret_type = None
10+
411
def __init__(self, *args):
5-
pass
12+
if not self.secret_type:
13+
raise ValueError('Plugins need to declare a secret_type.')
614

715
def analyze(self, file, filename): # pragma: no cover
816
"""
@@ -20,6 +28,7 @@ def analyze(self, file, filename): # pragma: no cover
2028

2129
return potential_secrets
2230

31+
@abstractmethod
2332
def analyze_string(self, string, line_num, filename): # pragma: no cover
2433
"""
2534
:param string: string; the line to analyze
@@ -29,10 +38,7 @@ def analyze_string(self, string, line_num, filename): # pragma: no cover
2938
3039
NOTE: line_num and filename are used for PotentialSecret creation only.
3140
"""
32-
33-
raise NotImplementedError(
34-
'%s needs to implement analyze_string()' % self.__class__.__name__
35-
)
41+
pass
3642

3743
@property
3844
def __dict__(self):

‎detect_secrets/plugins/high_entropy_strings.py‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@
55
import os
66
import re
77
import string
8+
from abc import ABCMeta
89
from contextlib import contextmanager
910

1011
import yaml
@@ -22,7 +23,7 @@
2223
class HighEntropyStringsPlugin(BasePlugin):
2324
"""Base class for string pattern matching"""
2425

25-
secret_type = 'High Entropy String'
26+
__metaclass__ = ABCMeta
2627

2728
def __init__(self, charset, limit, *args):
2829
if limit < 0 or limit > 8:

‎setup.py‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,13 @@
2020
'pyyaml',
2121
'unidiff',
2222
],
23-
extras_require={':python_version=="2.7"': ['configparser', 'enum34']},
23+
extras_require={
24+
':python_version=="2.7"': [
25+
'configparser',
26+
'enum34',
27+
'future',
28+
],
29+
},
2430
entry_points={
2531
'console_scripts': [
2632
'detect-secrets = detect_secrets.main:main',

0 commit comments

Comments
 (0)