Thanks to visit codestin.com
Credit goes to github.com

Skip to content

Commit a29108b

Browse files
KevinHockKevinHock
committed
🔭[Keyword Plugin] Precision improvements
Added Javascript specific false-positive checks Added ${ before } heuristic for e.g. ${link} Added more false-positives to FALSE_POSITIVES :zap: keyword_test.py Make STANDARD_NEGATIVES list and STANDARD_POSITIVES set for DRYness
1 parent d314550 commit a29108b

2 files changed

Lines changed: 133 additions & 83 deletions

File tree

‎detect_secrets/plugins/keyword.py‎

Lines changed: 43 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -48,18 +48,34 @@
4848
FALSE_POSITIVES = (
4949
"''",
5050
"''):",
51+
"')",
52+
"'this",
5153
'""',
5254
'""):',
55+
'")',
56+
'<a',
57+
'#pass',
58+
'#password',
59+
'<aws_secret_access_key>',
60+
'<password>',
61+
'dummy_secret',
5362
'false',
5463
'false):',
5564
'none',
5665
'none,',
5766
'none}',
5867
'not',
68+
'null,',
5969
'password)',
70+
'password,',
6071
'password},',
72+
'string,',
73+
'string}',
74+
'string}}',
75+
'test-access-key',
6176
'true',
6277
'true):',
78+
'{',
6379
)
6480
FOLLOWED_BY_COLON_RE = re.compile(
6581
# e.g. api_key: foo
@@ -104,8 +120,9 @@
104120

105121

106122
class FileType(Enum):
107-
PYTHON = 1
108-
PHP = 2
123+
JAVASCRIPT = 0
124+
PHP = 1
125+
PYTHON = 2
109126
OTHER = 3
110127

111128

@@ -115,7 +132,9 @@ def determine_file_type(filename):
115132
116133
:rtype: FileType
117134
"""
118-
if filename.endswith('.py'):
135+
if filename.endswith('.js'):
136+
return FileType.JAVASCRIPT
137+
elif filename.endswith('.py'):
119138
return FileType.PYTHON
120139
elif filename.endswith('.php'):
121140
return FileType.PHP
@@ -170,17 +189,23 @@ def secret_generator(self, string, filetype):
170189
def probably_false_positive(lowered_secret, filetype):
171190
if (
172191
'fake' in lowered_secret
173-
or 'self.' in lowered_secret
174-
or lowered_secret in FALSE_POSITIVES or
175-
# If it is a .php file, do not report $variables
176-
(
192+
or 'forgot' in lowered_secret
193+
or lowered_secret in FALSE_POSITIVES
194+
or (
195+
filetype == FileType.JAVASCRIPT
196+
and (
197+
lowered_secret.startswith('this.')
198+
or lowered_secret.startswith('fs.read')
199+
or lowered_secret == 'new'
200+
)
201+
) or ( # If it is a .php file, do not report $variables
177202
filetype == FileType.PHP
178203
and lowered_secret[0] == '$'
179204
)
180205
):
181206
return True
182207

183-
# No function calls
208+
# Heuristic for no function calls
184209
try:
185210
if (
186211
lowered_secret.index('(') < lowered_secret.index(')')
@@ -189,7 +214,7 @@ def probably_false_positive(lowered_secret, filetype):
189214
except ValueError:
190215
pass
191216

192-
# Skip over e.g. request.json_body['hey']
217+
# Heuristic for e.g. request.json_body['hey']
193218
try:
194219
if (
195220
lowered_secret.index('[') < lowered_secret.index(']')
@@ -198,4 +223,13 @@ def probably_false_positive(lowered_secret, filetype):
198223
except ValueError:
199224
pass
200225

226+
# Heuristic for e.g. ${link}
227+
try:
228+
if (
229+
lowered_secret.index('${') < lowered_secret.index('}')
230+
):
231+
return True
232+
except ValueError:
233+
pass
234+
201235
return False

‎tests/plugins/keyword_test.py‎

Lines changed: 90 additions & 74 deletions
Original file line numberDiff line numberDiff line change
@@ -8,40 +8,67 @@
88
from testing.mocks import mock_file_object
99

1010

11+
STANDARD_NEGATIVES = [
12+
# FOLLOWED_BY_COLON_RE
13+
'theapikey: ""', # Nothing in the quotes
14+
'theapikey: "somefakekey"', # 'fake' in the secret
15+
'theapikeyforfoo:hopenobodyfindsthisone', # Characters between apikey and :
16+
# FOLLOWED_BY_EQUAL_SIGNS_RE
17+
'some_key = "real_secret"', # We cannot make 'key' a Keyword, too noisy
18+
'my_password = foo(hey)you', # Has a ( followed by a )
19+
"my_password = request.json_body['hey']", # Has a [ followed by a ]
20+
'my_password = ""', # Nothing in the quotes
21+
"my_password = ''", # Nothing in the quotes
22+
'my_password = True', # 'True' is a known false-positive
23+
'my_password = "fakesecret"', # 'fake' in the secret
24+
'login(username=username, password=password)', # secret is password)
25+
'open(self, password = ""):', # secrets is ""):
26+
'open(self, password = ""):', # secrets is ""):
27+
# FOLLOWED_BY_QUOTES_AND_SEMICOLON_RE
28+
'private_key "";', # Nothing in the quotes
29+
'private_key \'"no spaces\';', # Has whitespace in the secret
30+
'private_key "fake";', # 'fake' in the secret
31+
'private_key "hopenobodyfindsthisone\';', # Double-quote does not match single-quote
32+
'private_key \'hopenobodyfindsthisone";', # Single-quote does not match double-quote
33+
'password: ${link}', # Has a ${ followed by a }
34+
]
35+
STANDARD_POSITIVES = {
36+
# FOLLOWED_BY_COLON_RE
37+
"'theapikey': 'h}o)p${e]nob(ody[finds>-_$#thisone'",
38+
'"theapikey": "h}o)p${e]nob(ody[finds>-_$#thisone"',
39+
'apikey: h}o)p${e]nob(ody[finds>-_$#thisone',
40+
'apikey:h}o)p${e]nob(ody[finds>-_$#thisone',
41+
'theapikey:h}o)p${e]nob(ody[finds>-_$#thisone',
42+
'apikey: "h}o)p${e]nob(ody[finds>-_$#thisone"',
43+
"apikey: 'h}o)p${e]nob(ody[finds>-_$#thisone'",
44+
# FOLLOWED_BY_EQUAL_SIGNS_RE
45+
'some_dict["secret"] = "h}o)p${e]nob(ody[finds>-_$#thisone"',
46+
"some_dict['secret'] = h}o)p${e]nob(ody[finds>-_$#thisone",
47+
'my_password=h}o)p${e]nob(ody[finds>-_$#thisone',
48+
'my_password= h}o)p${e]nob(ody[finds>-_$#thisone',
49+
'my_password =h}o)p${e]nob(ody[finds>-_$#thisone',
50+
'my_password = h}o)p${e]nob(ody[finds>-_$#thisone',
51+
'my_password =h}o)p${e]nob(ody[finds>-_$#thisone',
52+
'the_password=h}o)p${e]nob(ody[finds>-_$#thisone\n',
53+
'the_password= "h}o)p${e]nob(ody[finds>-_$#thisone"\n',
54+
'the_password=\'h}o)p${e]nob(ody[finds>-_$#thisone\'\n',
55+
# FOLLOWED_BY_QUOTES_AND_SEMICOLON_RE
56+
'apikey "h}o)p${e]nob(ody[finds>-_$#thisone";', # Double-quotes
57+
'fooapikeyfoo "h}o)p${e]nob(ody[finds>-_$#thisone";', # Double-quotes
58+
'fooapikeyfoo"h}o)p${e]nob(ody[finds>-_$#thisone";', # Double-quotes
59+
'private_key \'h}o)p${e]nob(ody[finds>-_$#thisone\';', # Single-quotes
60+
'fooprivate_keyfoo\'h}o)p${e]nob(ody[finds>-_$#thisone\';', # Single-quotes
61+
'fooprivate_key\'h}o)p${e]nob(ody[finds>-_$#thisone\';', # Single-quotes
62+
}
63+
64+
1165
class TestKeywordDetector(object):
1266

1367
@pytest.mark.parametrize(
1468
'file_content',
15-
[
16-
# FOLLOWED_BY_COLON_RE
17-
"'theapikey': 'ho)pe]nob(ody[finds>-_$#thisone'",
18-
'"theapikey": "ho)pe]nob(ody[finds>-_$#thisone"',
19-
'apikey: ho)pe]nob(ody[finds>-_$#thisone',
20-
'apikey:ho)pe]nob(ody[finds>-_$#thisone',
21-
'theapikey:ho)pe]nob(ody[finds>-_$#thisone',
22-
'apikey: "ho)pe]nob(ody[finds>-_$#thisone"',
23-
"apikey: 'ho)pe]nob(ody[finds>-_$#thisone'",
24-
# FOLLOWED_BY_EQUAL_SIGNS_RE
25-
'some_dict["secret"] = "ho)pe]nob(ody[finds>-_$#thisone"',
26-
"some_dict['secret'] = ho)pe]nob(ody[finds>-_$#thisone",
27-
'my_password=ho)pe]nob(ody[finds>-_$#thisone',
28-
'my_password= ho)pe]nob(ody[finds>-_$#thisone',
29-
'my_password =ho)pe]nob(ody[finds>-_$#thisone',
30-
'my_password = ho)pe]nob(ody[finds>-_$#thisone',
31-
'my_password =ho)pe]nob(ody[finds>-_$#thisone',
32-
'the_password=ho)pe]nob(ody[finds>-_$#thisone\n',
33-
'the_password= "ho)pe]nob(ody[finds>-_$#thisone"\n',
34-
'the_password=\'ho)pe]nob(ody[finds>-_$#thisone\'\n',
35-
# FOLLOWED_BY_QUOTES_AND_SEMICOLON_RE
36-
'apikey "ho)pe]nob(ody[finds>-_$#thisone";', # Double-quotes
37-
'fooapikeyfoo "ho)pe]nob(ody[finds>-_$#thisone";', # Double-quotes
38-
'fooapikeyfoo"ho)pe]nob(ody[finds>-_$#thisone";', # Double-quotes
39-
'private_key \'ho)pe]nob(ody[finds>-_$#thisone\';', # Single-quotes
40-
'fooprivate_keyfoo\'ho)pe]nob(ody[finds>-_$#thisone\';', # Single-quotes
41-
'fooprivate_key\'ho)pe]nob(ody[finds>-_$#thisone\';', # Single-quotes
42-
],
69+
STANDARD_POSITIVES,
4370
)
44-
def test_analyze_positives(self, file_content):
71+
def test_analyze_standard_positives(self, file_content):
4572
logic = KeywordDetector()
4673

4774
f = mock_file_object(file_content)
@@ -51,29 +78,25 @@ def test_analyze_positives(self, file_content):
5178
assert 'mock_filename' == potential_secret.filename
5279
assert (
5380
potential_secret.secret_hash
54-
== PotentialSecret.hash_secret('ho)pe]nob(ody[finds>-_$#thisone')
81+
== PotentialSecret.hash_secret('h}o)p${e]nob(ody[finds>-_$#thisone')
5582
)
5683

5784
@pytest.mark.parametrize(
5885
'file_content',
59-
[
86+
STANDARD_POSITIVES - {
6087
# FOLLOWED_BY_COLON_QUOTES_REQUIRED_RE
61-
"'theapikey': 'hope]nobody[finds>-_$#thisone'",
62-
'"theapikey": "hope]nobody[finds>-_$#thisone"',
63-
'apikey: "hope]nobody[finds>-_$#thisone"',
64-
"apikey: 'hope]nobody[finds>-_$#thisone'",
88+
'apikey: h}o)p${e]nob(ody[finds>-_$#thisone',
89+
'apikey:h}o)p${e]nob(ody[finds>-_$#thisone',
90+
'theapikey:h}o)p${e]nob(ody[finds>-_$#thisone',
6591
# FOLLOWED_BY_EQUAL_SIGNS_QUOTES_REQUIRED_RE
66-
'some_dict["secret"] = "hope]nobody[finds>-_$#thisone"',
67-
'the_password= "hope]nobody[finds>-_$#thisone"\n',
68-
'the_password=\'hope]nobody[finds>-_$#thisone\'\n',
69-
# FOLLOWED_BY_QUOTES_AND_SEMICOLON_RE
70-
'apikey "hope]nobody[finds>-_$#thisone";', # Double-quotes
71-
'fooapikeyfoo "hope]nobody[finds>-_$#thisone";', # Double-quotes
72-
'fooapikeyfoo"hope]nobody[finds>-_$#thisone";', # Double-quotes
73-
'private_key \'hope]nobody[finds>-_$#thisone\';', # Single-quotes
74-
'fooprivate_keyfoo\'hope]nobody[finds>-_$#thisone\';', # Single-quotes
75-
'fooprivate_key\'hope]nobody[finds>-_$#thisone\';', # Single-quotes
76-
],
92+
"some_dict['secret'] = h}o)p${e]nob(ody[finds>-_$#thisone",
93+
'my_password=h}o)p${e]nob(ody[finds>-_$#thisone',
94+
'my_password= h}o)p${e]nob(ody[finds>-_$#thisone',
95+
'my_password =h}o)p${e]nob(ody[finds>-_$#thisone',
96+
'my_password = h}o)p${e]nob(ody[finds>-_$#thisone',
97+
'my_password =h}o)p${e]nob(ody[finds>-_$#thisone',
98+
'the_password=h}o)p${e]nob(ody[finds>-_$#thisone\n',
99+
},
77100
)
78101
def test_analyze_python_positives(self, file_content):
79102
logic = KeywordDetector()
@@ -85,45 +108,38 @@ def test_analyze_python_positives(self, file_content):
85108
assert 'mock_filename.py' == potential_secret.filename
86109
assert (
87110
potential_secret.secret_hash
88-
== PotentialSecret.hash_secret('hope]nobody[finds>-_$#thisone')
111+
== PotentialSecret.hash_secret('h}o)p${e]nob(ody[finds>-_$#thisone')
89112
)
90113

91114
@pytest.mark.parametrize(
92-
'file_content',
93-
[
115+
'negative',
116+
STANDARD_NEGATIVES,
117+
)
118+
def test_analyze_standard_negatives(self, negative):
119+
logic = KeywordDetector()
120+
121+
f = mock_file_object(negative)
122+
output = logic.analyze(f, 'mock_filename.foo')
123+
assert len(output) == 0
124+
125+
@pytest.mark.parametrize(
126+
'js_negative',
127+
STANDARD_NEGATIVES + [
94128
# FOLLOWED_BY_COLON_RE
95-
'private_key "";', # Nothing in the quotes
96-
'private_key \'"no spaces\';', # Has whitespace in the secret
97-
'private_key "fake";', # 'fake' in the secret
98-
'private_key "hopenobodyfindsthisone\';', # Double-quote does not match single-quote
99-
'private_key \'hopenobodyfindsthisone";', # Single-quote does not match double-quote
100-
# FOLLOWED_BY_QUOTES_AND_SEMICOLON_RE
101-
'theapikey: ""', # Nothing in the quotes
102-
'theapikey: "somefakekey"', # 'fake' in the secret
103-
'theapikeyforfoo:hopenobodyfindsthisone', # Characters between apikey and :
104-
# FOLLOWED_BY_EQUAL_SIGNS_RE
105-
'some_key = "real_secret"', # We cannot make 'key' a Keyword, too noisy
106-
'my_password = foo(hey)you', # Has a ( followed by a )
107-
"my_password = request.json_body['hey']", # Has a [ followed by a ]
108-
'my_password = ""', # Nothing in the quotes
109-
"my_password = ''", # Nothing in the quotes
110-
'my_password = True', # 'True' is a known false-positive
111-
'my_password = "fakesecret"', # 'fake' in the secret
112-
'login(username=username, password=password)', # secret is password)
113-
'open(self, password = ""):', # secrets is ""):
114-
# '',
129+
'apiKey: this.apiKey,',
130+
"apiKey: fs.readFileSync('foo',",
115131
],
116132
)
117-
def test_analyze_negatives(self, file_content):
133+
def test_analyze_javascript_negatives(self, js_negative):
118134
logic = KeywordDetector()
119135

120-
f = mock_file_object(file_content)
121-
output = logic.analyze(f, 'mock_filename.foo')
136+
f = mock_file_object(js_negative)
137+
output = logic.analyze(f, 'mock_filename.js')
122138
assert len(output) == 0
123139

124140
@pytest.mark.parametrize(
125141
'secret_starting_with_dollar_sign',
126-
[
142+
STANDARD_NEGATIVES + [
127143
# FOLLOWED_BY_EQUAL_SIGNS_RE
128144
'$password = $input;',
129145
],
@@ -137,7 +153,7 @@ def test_analyze_php_negatives(self, secret_starting_with_dollar_sign):
137153

138154
@pytest.mark.parametrize(
139155
'secret_with_no_quote',
140-
[
156+
STANDARD_NEGATIVES + [
141157
# FOLLOWED_BY_COLON_QUOTES_REQUIRED_RE
142158
'apikey: hope]nobody[finds>-_$#thisone',
143159
'apikey:hope]nobody[finds>-_$#thisone',

0 commit comments

Comments
 (0)