Thanks to visit codestin.com
Credit goes to github.com

Skip to content

Commit e1f86c9

Browse files
stamparmstamparm
committed
minor refactoring
1 parent bcf9fc6 commit e1f86c9

8 files changed

Lines changed: 176 additions & 154 deletions

File tree

‎lib/core/bigarray.py‎

Lines changed: 99 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,99 @@
1+
#!/usr/bin/env python
2+
3+
"""
4+
$Id$
5+
6+
Copyright (c) 2006-2012 sqlmap developers (http://www.sqlmap.org/)
7+
See the file 'doc/COPYING' for copying permission
8+
"""
9+
10+
import pickle
11+
import tempfile
12+
13+
from lib.core.settings import BIGARRAY_CHUNK_LENGTH
14+
15+
class BigArray(list):
16+
"""
17+
List-like object used for storing large amounts of data (disk cached)
18+
"""
19+
20+
def __init__(self):
21+
self.chunks = [[]]
22+
self.cache = None
23+
self.length = 0
24+
self.filenames = set()
25+
26+
def append(self, value):
27+
self.chunks[-1].append(value)
28+
if len(self.chunks[-1]) >= BIGARRAY_CHUNK_LENGTH:
29+
filename = self._dump(self.chunks[-1])
30+
del(self.chunks[-1][:])
31+
self.chunks[-1] = filename
32+
self.chunks.append([])
33+
34+
def pop(self):
35+
if len(self.chunks[-1]) < 1:
36+
self.chunks.pop()
37+
with open(self.chunks[-1], 'rb') as fp:
38+
self.chunks[-1] = pickle.load(fp)
39+
return self.chunks[-1].pop()
40+
41+
def index(self, value):
42+
for index in xrange(len(self)):
43+
if self[index] == value:
44+
return index
45+
return ValueError, "%s is not in list" % value
46+
47+
def _dump(self, value):
48+
handle, filename = tempfile.mkstemp()
49+
self.filenames.add(filename)
50+
os.close(handle)
51+
with open(filename, 'w+b') as fp:
52+
pickle.dump(value, fp)
53+
return filename
54+
55+
def _checkcache(self, index):
56+
if (self.cache and self.cache[0] != index and self.cache[2]):
57+
filename = self._dump(self.cache[1])
58+
self.chunks[self.cache[0]] = filename
59+
if not (self.cache and self.cache[0] == index):
60+
with open(self.chunks[index], 'rb') as fp:
61+
self.cache = (index, pickle.load(fp), False)
62+
63+
def __getitem__(self, y):
64+
index = y / BIGARRAY_CHUNK_LENGTH
65+
offset = y % BIGARRAY_CHUNK_LENGTH
66+
chunk = self.chunks[index]
67+
if isinstance(chunk, list):
68+
return chunk[offset]
69+
else:
70+
self._checkcache(index)
71+
return self.cache[1][offset]
72+
73+
def __setitem__(self, y, value):
74+
index = y / BIGARRAY_CHUNK_LENGTH
75+
offset = y % BIGARRAY_CHUNK_LENGTH
76+
chunk = self.chunks[index]
77+
if isinstance(chunk, list):
78+
chunk[offset] = value
79+
else:
80+
self._checkcache(index)
81+
self.cache[1][offset] = value
82+
self.cache[2] = True # dirty flag
83+
84+
def __repr__(self):
85+
return "%s%s" % ("..." if len(self.chunks) > 1 else "", self.chunks[-1].__repr__())
86+
87+
def __iter__(self):
88+
for i in xrange(len(self)):
89+
yield self[i]
90+
91+
def __len__(self):
92+
return len(self.chunks[-1]) if len(self.chunks) == 1 else (len(self.chunks) - 1) * BIGARRAY_CHUNK_LENGTH + len(self.chunks[-1])
93+
94+
def __del__(self):
95+
for filename in self.filenames:
96+
try:
97+
os.remove(filename)
98+
except OSError:
99+
pass

‎lib/core/common.py‎

Lines changed: 1 addition & 149 deletions
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,6 @@
2424
import string
2525
import struct
2626
import sys
27-
import tempfile
2827
import time
2928
import types
3029
import urllib
@@ -50,6 +49,7 @@
5049
from extra.cloak.cloak import decloak
5150
from extra.magic import magic
5251
from extra.odict.odict import OrderedDict
52+
from lib.core.bigarray import BigArray
5353
from lib.core.data import conf
5454
from lib.core.data import kb
5555
from lib.core.data import logger
@@ -76,7 +76,6 @@
7676
from lib.core.exception import sqlmapSilentQuitException
7777
from lib.core.exception import sqlmapSyntaxException
7878
from lib.core.optiondict import optDict
79-
from lib.core.settings import BIGARRAY_CHUNK_LENGTH
8079
from lib.core.settings import DEFAULT_COOKIE_DELIMITER
8180
from lib.core.settings import DEFAULT_GET_POST_DELIMITER
8281
from lib.core.settings import DUMMY_USER_INJECTION
@@ -156,153 +155,6 @@ def write(self, fp):
156155

157156
fp.write("\n")
158157

159-
class Wordlist:
160-
"""
161-
Iterator for looping over a large dictionaries
162-
"""
163-
164-
def __init__(self, filenames):
165-
self.filenames = filenames
166-
self.fp = None
167-
self.index = 0
168-
self.iter = None
169-
self.custom = []
170-
self.adjust()
171-
self.lock = None
172-
173-
def __iter__(self):
174-
return self
175-
176-
def adjust(self):
177-
self.closeFP()
178-
if self.index > len(self.filenames):
179-
raise StopIteration
180-
elif self.index == len(self.filenames):
181-
if self.custom:
182-
self.iter = iter(self.custom)
183-
else:
184-
raise StopIteration
185-
else:
186-
current = self.filenames[self.index]
187-
infoMsg = "loading dictionary from '%s'" % current
188-
singleTimeLogMessage(infoMsg)
189-
self.fp = open(current, "r")
190-
self.iter = iter(self.fp)
191-
192-
self.index += 1
193-
194-
def append(self, value):
195-
self.custom.append(value)
196-
197-
def closeFP(self):
198-
if self.fp:
199-
self.fp.close()
200-
self.fp = None
201-
202-
def next(self):
203-
retVal = None
204-
if self.lock:
205-
self.lock.acquire()
206-
try:
207-
retVal = self.iter.next().rstrip()
208-
except StopIteration:
209-
self.adjust()
210-
retVal = self.iter.next().rstrip()
211-
finally:
212-
if self.lock:
213-
self.lock.release()
214-
return retVal
215-
216-
def rewind(self):
217-
self.index = 0
218-
self.adjust()
219-
220-
class BigArray(list):
221-
"""
222-
List-like object used for storing large amounts of data (disk cached)
223-
"""
224-
225-
def __init__(self):
226-
self.chunks = [[]]
227-
self.cache = None
228-
self.length = 0
229-
self.filenames = set()
230-
231-
def append(self, value):
232-
self.chunks[-1].append(value)
233-
if len(self.chunks[-1]) >= BIGARRAY_CHUNK_LENGTH:
234-
filename = self._dump(self.chunks[-1])
235-
del(self.chunks[-1][:])
236-
self.chunks[-1] = filename
237-
self.chunks.append([])
238-
239-
def pop(self):
240-
if len(self.chunks[-1]) < 1:
241-
self.chunks.pop()
242-
with open(self.chunks[-1], 'rb') as fp:
243-
self.chunks[-1] = pickle.load(fp)
244-
return self.chunks[-1].pop()
245-
246-
def index(self, value):
247-
for index in xrange(len(self)):
248-
if self[index] == value:
249-
return index
250-
return ValueError, "%s is not in list" % value
251-
252-
def _dump(self, value):
253-
handle, filename = tempfile.mkstemp()
254-
self.filenames.add(filename)
255-
os.close(handle)
256-
with open(filename, 'w+b') as fp:
257-
pickle.dump(value, fp)
258-
return filename
259-
260-
def _checkcache(self, index):
261-
if (self.cache and self.cache[0] != index and self.cache[2]):
262-
filename = self._dump(self.cache[1])
263-
self.chunks[self.cache[0]] = filename
264-
if not (self.cache and self.cache[0] == index):
265-
with open(self.chunks[index], 'rb') as fp:
266-
self.cache = (index, pickle.load(fp), False)
267-
268-
def __getitem__(self, y):
269-
index = y / BIGARRAY_CHUNK_LENGTH
270-
offset = y % BIGARRAY_CHUNK_LENGTH
271-
chunk = self.chunks[index]
272-
if isinstance(chunk, list):
273-
return chunk[offset]
274-
else:
275-
self._checkcache(index)
276-
return self.cache[1][offset]
277-
278-
def __setitem__(self, y, value):
279-
index = y / BIGARRAY_CHUNK_LENGTH
280-
offset = y % BIGARRAY_CHUNK_LENGTH
281-
chunk = self.chunks[index]
282-
if isinstance(chunk, list):
283-
chunk[offset] = value
284-
else:
285-
self._checkcache(index)
286-
self.cache[1][offset] = value
287-
self.cache[2] = True # dirty flag
288-
289-
def __repr__(self):
290-
return "%s%s" % ("..." if len(self.chunks) > 1 else "", self.chunks[-1].__repr__())
291-
292-
def __iter__(self):
293-
for i in xrange(len(self)):
294-
yield self[i]
295-
296-
def __len__(self):
297-
return len(self.chunks[-1]) if len(self.chunks) == 1 else (len(self.chunks) - 1) * BIGARRAY_CHUNK_LENGTH + len(self.chunks[-1])
298-
299-
def __del__(self):
300-
for filename in self.filenames:
301-
try:
302-
os.remove(filename)
303-
except OSError:
304-
pass
305-
306158
class DynamicContentItem:
307159
"""
308160
Represents line in content page with dynamic properties (candidate

‎lib/core/wordlist.py‎

Lines changed: 71 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,71 @@
1+
#!/usr/bin/env python
2+
3+
"""
4+
$Id$
5+
6+
Copyright (c) 2006-2012 sqlmap developers (http://www.sqlmap.org/)
7+
See the file 'doc/COPYING' for copying permission
8+
"""
9+
10+
from lib.core.common import singleTimeLogMessage
11+
12+
class Wordlist:
13+
"""
14+
Iterator for looping over a large dictionaries
15+
"""
16+
17+
def __init__(self, filenames):
18+
self.filenames = filenames
19+
self.fp = None
20+
self.index = 0
21+
self.iter = None
22+
self.custom = []
23+
self.adjust()
24+
self.lock = None
25+
26+
def __iter__(self):
27+
return self
28+
29+
def adjust(self):
30+
self.closeFP()
31+
if self.index > len(self.filenames):
32+
raise StopIteration
33+
elif self.index == len(self.filenames):
34+
if self.custom:
35+
self.iter = iter(self.custom)
36+
else:
37+
raise StopIteration
38+
else:
39+
current = self.filenames[self.index]
40+
infoMsg = "loading dictionary from '%s'" % current
41+
singleTimeLogMessage(infoMsg)
42+
self.fp = open(current, "r")
43+
self.iter = iter(self.fp)
44+
45+
self.index += 1
46+
47+
def append(self, value):
48+
self.custom.append(value)
49+
50+
def closeFP(self):
51+
if self.fp:
52+
self.fp.close()
53+
self.fp = None
54+
55+
def next(self):
56+
retVal = None
57+
if self.lock:
58+
self.lock.acquire()
59+
try:
60+
retVal = self.iter.next().rstrip()
61+
except StopIteration:
62+
self.adjust()
63+
retVal = self.iter.next().rstrip()
64+
finally:
65+
if self.lock:
66+
self.lock.release()
67+
return retVal
68+
69+
def rewind(self):
70+
self.index = 0
71+
self.adjust()

‎lib/request/inject.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -11,8 +11,8 @@
1111
import time
1212

1313
from lib.core.agent import agent
14+
from lib.core.bigarray import BigArray
1415
from lib.core.common import Backend
15-
from lib.core.common import BigArray
1616
from lib.core.common import calculateDeltaSeconds
1717
from lib.core.common import cleanQuery
1818
from lib.core.common import dataToSessionFile

‎lib/techniques/error/use.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -12,8 +12,8 @@
1212
import time
1313

1414
from lib.core.agent import agent
15+
from lib.core.bigarray import BigArray
1516
from lib.core.common import Backend
16-
from lib.core.common import BigArray
1717
from lib.core.common import calculateDeltaSeconds
1818
from lib.core.common import dataToStdout
1919
from lib.core.common import extractRegexResult

‎lib/techniques/union/use.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -12,8 +12,8 @@
1212
import time
1313

1414
from lib.core.agent import agent
15+
from lib.core.bigarray import BigArray
1516
from lib.core.common import Backend
16-
from lib.core.common import BigArray
1717
from lib.core.common import calculateDeltaSeconds
1818
from lib.core.common import clearConsoleLine
1919
from lib.core.common import dataToStdout

‎lib/utils/hash.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -46,7 +46,6 @@
4646
from lib.core.common import readInput
4747
from lib.core.common import singleTimeLogMessage
4848
from lib.core.common import singleTimeWarnMessage
49-
from lib.core.common import Wordlist
5049
from lib.core.convert import hexdecode
5150
from lib.core.convert import hexencode
5251
from lib.core.convert import utf8encode
@@ -68,6 +67,7 @@
6867
from lib.core.settings import ML
6968
from lib.core.settings import UNICODE_ENCODING
7069
from lib.core.settings import ROTATING_CHARS
70+
from lib.core.wordlist import Wordlist
7171

7272
def mysql_passwd(password, uppercase=True):
7373
"""

0 commit comments

Comments
 (0)