Thanks to visit codestin.com
Credit goes to github.com

Skip to content

Commit ec1bc02

Browse files
stamparmstamparm
committed
hello big tables, this is sqlmap, sqlmap this is big tables
1 parent 82e1e61 commit ec1bc02

8 files changed

Lines changed: 108 additions & 26 deletions

File tree

‎lib/core/common.py‎

Lines changed: 76 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,9 @@
2323
import string
2424
import struct
2525
import sys
26+
import tempfile
2627
import time
28+
import types
2729
import urlparse
2830
import unicodedata
2931

@@ -205,31 +207,86 @@ def __init__(self):
205207
self.chunks = [[]]
206208
self.cache = None
207209
self.length = 0
210+
self.filenames = set()
208211

209212
def append(self, value):
210213
self.chunks[-1].append(value)
211214
if len(self.chunks[-1]) >= BIGARRAY_CHUNK_LENGTH:
212-
fp = tempfile.TemporaryFile()
213-
pickle.dump(self.chunks[-1], fp)
215+
filename = self._dump(self.chunks[-1])
214216
del(self.chunks[-1][:])
215-
self.chunks[-1] = fp
217+
self.chunks[-1] = filename
216218
self.chunks.append([])
217219

220+
def pop(self):
221+
if len(self.chunks[-1]) < 1:
222+
self.chunks.pop()
223+
fp = open(self.chunks[-1], 'rb')
224+
self.chunks[-1] = pickle.load(fp)
225+
fp.close()
226+
return self.chunks[-1].pop()
227+
228+
def index(self, value):
229+
for index in xrange(len(self)):
230+
if self[index] == value:
231+
return index
232+
return ValueError, "%s is not in list" % value
233+
234+
def _dump(self, value):
235+
handle, filename = tempfile.mkstemp()
236+
self.filenames.add(filename)
237+
os.close(handle)
238+
fp = open(filename, 'w+b')
239+
pickle.dump(value, fp)
240+
fp.close()
241+
return filename
242+
243+
def _checkcache(self, index):
244+
if (self.cache and self.cache[0] != index and self.cache[2]):
245+
filename = self._dump(self.cache[1])
246+
self.chunks[self.cache[0]] = filename
247+
if not (self.cache and self.cache[0] == index):
248+
fp = open(self.chunks[index], 'rb')
249+
self.cache = [index, pickle.load(fp), False]
250+
fp.close()
251+
218252
def __getitem__(self, y):
219253
index = y / BIGARRAY_CHUNK_LENGTH
220254
offset = y % BIGARRAY_CHUNK_LENGTH
221255
chunk = self.chunks[index]
222256
if isinstance(chunk, list):
223257
return chunk[offset]
224258
else:
225-
if not (self.cache and self.cache[0] == index):
226-
chunk.seek(0)
227-
self.cache = (index, pickle.load(chunk))
259+
self._checkcache(index)
228260
return self.cache[1][offset]
229261

262+
def __setitem__(self, y, value):
263+
index = y / BIGARRAY_CHUNK_LENGTH
264+
offset = y % BIGARRAY_CHUNK_LENGTH
265+
chunk = self.chunks[index]
266+
if isinstance(chunk, list):
267+
chunk[offset] = value
268+
else:
269+
self._checkcache(index)
270+
self.cache[1][offset] = value
271+
self.cache[2] = True # dirty flag
272+
273+
def __repr__(self):
274+
return "%s%s" % ("..." if len(self.chunks) > 1 else "", self.chunks[-1].__repr__())
275+
276+
def __iter__(self):
277+
for i in xrange(len(self)):
278+
yield self[i]
279+
230280
def __len__(self):
231281
return len(self.chunks[-1]) if len(self.chunks) == 1 else (len(self.chunks) - 1) * BIGARRAY_CHUNK_LENGTH + len(self.chunks[-1])
232282

283+
def __del__(self):
284+
for filename in self.filenames:
285+
try:
286+
os.remove(filename)
287+
except OSError:
288+
pass
289+
233290
class DynamicContentItem:
234291
"""
235292
Represents line in content page with dynamic properties (candidate
@@ -561,6 +618,15 @@ def isVersionGreaterOrEqualThan(version):
561618
def isOs(os):
562619
return Backend.getOs() is not None and Backend.getOs().lower() == os.lower()
563620

621+
# Reference: http://code.activestate.com/recipes/325205-cache-decorator-in-python-24/
622+
def cachedmethod(f, cache={}):
623+
def g(*args, **kwargs):
624+
key = ( f, tuple(args), frozenset(kwargs.items()) )
625+
if key not in cache:
626+
cache[key] = f(*args, **kwargs)
627+
return cache[key]
628+
return g
629+
564630
def paramToDict(place, parameters=None):
565631
"""
566632
Split the parameters into names and values, check if these parameters
@@ -1266,7 +1332,7 @@ def parseUnionPage(output, expression, partial=False, condition=None, sort=True)
12661332
if output is None:
12671333
return None
12681334

1269-
data = []
1335+
data = BigArray()
12701336

12711337
outCond1 = ( output.startswith(kb.misc.start) and output.endswith(kb.misc.stop) )
12721338
outCond2 = ( output.startswith(DUMP_START_MARKER) and output.endswith(DUMP_STOP_MARKER) )
@@ -2204,6 +2270,7 @@ def isNumPosStrValue(value):
22042270

22052271
return value and isinstance(value, basestring) and value.isdigit() and value != "0"
22062272

2273+
@cachedmethod
22072274
def aliasToDbmsEnum(dbms):
22082275
"""
22092276
Returns major DBMS name from a given alias
@@ -2730,8 +2797,8 @@ def isNoneValue(value):
27302797
if len(value) == 1:
27312798
return isNoneValue(value[0])
27322799
else:
2733-
for i in xrange(len(value)):
2734-
if value[i] and value[i] != "None":
2800+
for item in value:
2801+
if item and item != "None":
27352802
return False
27362803
return True
27372804
elif isinstance(value, dict):

‎lib/core/dump.py‎

Lines changed: 12 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@
2424
from lib.core.data import logger
2525
from lib.core.enums import DBMS
2626
from lib.core.replication import Replication
27+
from lib.core.settings import TRIM_STDOUT_DUMP_SIZE
2728
from lib.core.settings import UNICODE_ENCODING
2829

2930
class Dump:
@@ -37,9 +38,10 @@ def __init__(self):
3738
self.__outputFile = None
3839
self.__outputFP = None
3940

40-
def __write(self, data, n=True):
41+
def __write(self, data, n=True, console=True):
4142
text = "%s%s" % (data, "\n" if n else " ")
42-
dataToStdout(text)
43+
if console:
44+
dataToStdout(text)
4345

4446
self.__outputFP.write(text)
4547
self.__outputFP.flush()
@@ -407,7 +409,13 @@ def dbTableValues(self, tableValues):
407409
if conf.replicate:
408410
rtable.beginTransaction()
409411

412+
if count > TRIM_STDOUT_DUMP_SIZE:
413+
warnMsg = "console output will be trimmed "
414+
warnMsg += "due to the large table size"
415+
logger.warning(warnMsg)
416+
410417
for i in range(count):
418+
console = (i >= count - TRIM_STDOUT_DUMP_SIZE)
411419
field = 1
412420
values = []
413421

@@ -429,7 +437,7 @@ def dbTableValues(self, tableValues):
429437
values.append(value)
430438
maxlength = int(info["length"])
431439
blank = " " * (maxlength - len(value))
432-
self.__write("| %s%s" % (value, blank), n=False)
440+
self.__write("| %s%s" % (value, blank), n=False, console=console)
433441

434442
if not conf.replicate:
435443
if not conf.multipleTargets and field == fields:
@@ -442,7 +450,7 @@ def dbTableValues(self, tableValues):
442450
if conf.replicate:
443451
rtable.insert(values)
444452

445-
self.__write("|")
453+
self.__write("|", console=console)
446454

447455
if not conf.multipleTargets and not conf.replicate:
448456
dataToDumpFile(dumpFP, "\n")

‎lib/core/profiling.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -31,7 +31,7 @@ def profile(profileOutputFile=None, dotOutputFile=None, imageOutputFile=None):
3131
errMsg = "profiling requires third-party libraries (%s). " % getUnicode(e, UNICODE_ENCODING)
3232
errMsg += "Quick steps:%s" % os.linesep
3333
errMsg += "1) Install http://code.google.com/p/pydot/%s" % os.linesep
34-
errMsg += "2) sudo apt-get install python-profiler graphviz"
34+
errMsg += "2) sudo apt-get install python-pyparsing python-profiler graphviz"
3535
logger.error(errMsg)
3636

3737
return

‎lib/core/settings.py‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -383,5 +383,8 @@
383383
# Used for status representation in dictionary attack phase
384384
ROTATING_CHARS = ('\\', '|', '|', '/', '-')
385385

386-
# Chunk length used in BigArray object (only last one is held in memory)
387-
BIGARRAY_CHUNK_LENGTH = 10000
386+
# Chunk length (in items) used by BigArray objects (only last chunk and cached one are held in memory)
387+
BIGARRAY_CHUNK_LENGTH = 5000
388+
389+
# Only console display last n table rows
390+
TRIM_STDOUT_DUMP_SIZE = 256

‎lib/request/inject.py‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@
1212

1313
from lib.core.agent import agent
1414
from lib.core.common import Backend
15+
from lib.core.common import BigArray
1516
from lib.core.common import calculateDeltaSeconds
1617
from lib.core.common import cleanQuery
1718
from lib.core.common import dataToSessionFile
@@ -123,7 +124,7 @@ def __goInferenceProxy(expression, fromUser=False, expected=None, batch=False, r
123124
count = None
124125
startLimit = 0
125126
stopLimit = None
126-
outputs = []
127+
outputs = BigArray()
127128
test = None
128129
untilLimitChar = None
129130
untilOrderChar = None

‎lib/techniques/error/use.py‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@
1313

1414
from lib.core.agent import agent
1515
from lib.core.common import Backend
16+
from lib.core.common import BigArray
1617
from lib.core.common import calculateDeltaSeconds
1718
from lib.core.common import dataToSessionFile
1819
from lib.core.common import dataToStdout
@@ -321,7 +322,7 @@ def errorUse(expression, expected=None, resumeValue=True, dump=False):
321322
threadData = getCurrentThreadData()
322323
threadData.shared.limits = range(startLimit, stopLimit)
323324
numThreads = min(conf.threads, len(threadData.shared.limits))
324-
threadData.shared.outputs = []
325+
threadData.shared.outputs = BigArray()
325326

326327
if stopLimit > TURN_OFF_RESUME_INFO_LIMIT:
327328
kb.suppressResumeInfo = True

‎lib/utils/hash.py‎

Lines changed: 7 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -33,13 +33,12 @@
3333

3434
from extra.pydes.pyDes import des
3535
from extra.pydes.pyDes import CBC
36+
from lib.core.common import Backend
3637
from lib.core.common import checkFile
3738
from lib.core.common import clearConsoleLine
3839
from lib.core.common import dataToStdout
3940
from lib.core.common import getCompiledRegex
4041
from lib.core.common import getFileItems
41-
from lib.core.common import Backend
42-
from lib.core.common import getCompiledRegex
4342
from lib.core.common import getPublicTypeMembers
4443
from lib.core.common import normalizeUnicode
4544
from lib.core.common import paths
@@ -252,6 +251,8 @@ def attackCachedUsersPasswords():
252251
kb.data.cachedUsersPasswords[user][i] += "%s clear-text password: %s" % ('\n' if kb.data.cachedUsersPasswords[user][i][-1] != '\n' else '', password)
253252

254253
def attackDumpedTable():
254+
isOracle, isMySQL = Backend.isDbms(DBMS.ORACLE), Backend.isDbms(DBMS.MYSQL)
255+
255256
if kb.data.dumpedTable:
256257
table = kb.data.dumpedTable
257258
columns = table.keys()
@@ -275,7 +276,7 @@ def attackDumpedTable():
275276

276277
value = table[column]['values'][i]
277278

278-
if hashRecognition(value):
279+
if hashRecognition(value, isOracle, isMySQL):
279280
if colUser:
280281
if table[colUser]['values'][i] not in attack_dict:
281282
attack_dict[table[colUser]['values'][i]] = []
@@ -310,15 +311,15 @@ def attackDumpedTable():
310311
table[column]['values'][i] += " (%s)" % password
311312
table[column]['length'] = max(table[column]['length'], len(table[column]['values'][i]))
312313

313-
def hashRecognition(value):
314+
def hashRecognition(value, isOracle=False, isMySQL=False):
314315
retVal = None
315316

316317
if isinstance(value, basestring):
317318
for name, regex in getPublicTypeMembers(HASH):
318319
# Hashes for Oracle and old MySQL look the same hence these checks
319-
if Backend.isDbms(DBMS.ORACLE) and regex == HASH.MYSQL_OLD:
320+
if isOracle and regex == HASH.MYSQL_OLD:
320321
continue
321-
elif Backend.isDbms(DBMS.MYSQL) and regex == HASH.ORACLE_OLD:
322+
elif isMySQL and regex == HASH.ORACLE_OLD:
322323
continue
323324
elif regex == HASH.CRYPT_GENERIC:
324325
if any([getCompiledRegex(GENERAL_IP_ADDRESS_REGEX).match(value), value.lower() == value, value.upper() == value, value.isdigit()]):

‎plugins/generic/enumeration.py‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@
1313
from lib.core.agent import agent
1414
from lib.core.common import arrayizeValue
1515
from lib.core.common import Backend
16+
from lib.core.common import BigArray
1617
from lib.core.common import clearConsoleLine
1718
from lib.core.common import dataToStdout
1819
from lib.core.common import getRange
@@ -1385,7 +1386,7 @@ def __pivotDumpTable(self, table, colList, count=None, blind=True):
13851386

13861387
for column in colList:
13871388
lengths[column] = 0
1388-
entries[column] = []
1389+
entries[column] = BigArray()
13891390

13901391
colList = sorted(colList, key=lambda x: len(x) if x else MAX_INT)
13911392

@@ -1706,7 +1707,7 @@ def dumpTable(self, foundData=None):
17061707
lengths[column] = 0
17071708

17081709
if column not in entries:
1709-
entries[column] = []
1710+
entries[column] = BigArray()
17101711

17111712
if Backend.getIdentifiedDbms() in ( DBMS.MYSQL, DBMS.PGSQL ):
17121713
query = rootQuery.blind.query % (column, conf.db, conf.tbl, index)

0 commit comments

Comments
 (0)