Thanks to visit codestin.com
Credit goes to github.com

Skip to content

Commit e010084

Browse files
ByronByron
committed
test are running, once again, but not yet complete regarding the static manager
1 parent 631b9ea commit e010084

2 files changed

Lines changed: 70 additions & 93 deletions

File tree

‎smmap/mman.py‎

Lines changed: 31 additions & 61 deletions
Original file line numberDiff line numberDiff line change
@@ -95,7 +95,8 @@ def use_region(self, offset, size, flags = 0):
9595
either the file has reached its end, or the map was created between two existing regions"""
9696
need_region = True
9797
man = self._manager
98-
size = min(size, man.window_size()) # clamp size to window size
98+
fsize = self._rlist.file_size()
99+
size = min(size, man.window_size() or fsize) # clamp size to window size
99100

100101
if self._region is not None:
101102
if self._region.includes_ofs(offset):
@@ -106,7 +107,7 @@ def use_region(self, offset, size, flags = 0):
106107
# END check existing region
107108

108109
# offset too large ?
109-
if offset >= self._rlist.file_size():
110+
if offset >= fsize:
110111
return self
111112
#END handle offset
112113

@@ -238,10 +239,11 @@ class StaticWindowMapManager(object):
238239

239240
_MB_in_bytes = 1024 * 1024
240241

241-
def __init__(self, window_size = sys.maxint, max_memory_size = 0, max_open_handles = sys.maxint):
242+
def __init__(self, window_size = 0, max_memory_size = 0, max_open_handles = sys.maxint):
242243
"""initialize the manager with the given parameters.
243-
:param window_size: if 0, a default window size will be chosen depending on
244+
:param window_size: if -1, a default window size will be chosen depending on
244245
the operating system's architechture. It will internally be quantified to a multiple of the page size
246+
If 0, the window may have any size, which basically results in mapping the whole file at one
245247
:param max_memory_size: maximum amount of memory we may map at once before releasing mapped regions.
246248
If 0, a viable default iwll be set dependning on the system's architecture.
247249
:param max_open_handles: if not maxin, limit the amount of open file handles to the given number.
@@ -254,7 +256,7 @@ def __init__(self, window_size = sys.maxint, max_memory_size = 0, max_open_handl
254256
self._memory_size = 0
255257
self._handle_count = 0
256258

257-
if window_size == 0:
259+
if window_size < 0:
258260
coeff = 32
259261
if is_64_bit():
260262
coeff = 1024
@@ -281,43 +283,40 @@ def _collect_lru_region(self, size):
281283
:todo: implement a case where all unusued regions are discarded efficiently. Currently its only brute force"""
282284
num_found = 0
283285
while (size == 0) or (self._memory_size + size > self._max_memory_size):
284-
for k, regions in self._fdict.iteritems():
285-
found_lonely_region = False
286+
lru_region = None
287+
lru_list = None
288+
for regions in self._fdict.itervalues():
286289
for region in regions:
287290
# check client count - consider that we keep one reference ourselves !
288291
if (region.client_count()-2 == 0 and
289292
(lru_region is None or region._uc < lru_region._uc)):
290-
# remove whole list
291-
found_lonely_region = True
292-
num_found += 1
293-
self._memory_size -= region.size()
294-
self._handle_count -= 1
295-
self._fdict.pop(k)
296-
297-
break
293+
lru_region = region
294+
lru_list = regions
298295
# END update lru_region
299296
#END for each region
300-
if found_lonely_region:
301-
continue
302-
# END skip iteration and restart
303297
#END for each regions list
304298

305-
# still here ?
306-
if num_found == 0 and size != 0:
307-
raise RegionCollectionError("Didn't find any region to free")
308-
#END raise if necessary
299+
if lru_region is None:
300+
if num_found == 0 and size != 0:
301+
raise RegionCollectionError("Didn't find any region to free")
302+
#END raise if necessary
303+
break
304+
#END handle region not found
305+
306+
num_found += 1
307+
del(lru_list[lru_list.index(lru_region)])
308+
self._memory_size -= lru_region.size()
309+
self._handle_count -= 1
309310
#END while there is more memory to free
310-
311311
return num_found
312312

313-
314313
def _obtain_region(self, a, offset, size, flags, is_recursive):
315314
"""Utilty to create a new region - for more information on the parameters,
316315
see MapCursor.use_region.
317316
:param a: A regions (a)rray
318317
:return: The newly created region"""
319-
if self._memory_size + window_size > self._max_memory_size:
320-
self._collect_lru_region(window_size)
318+
if self._memory_size + size > self._max_memory_size:
319+
self._collect_lru_region(size)
321320
#END handle collection
322321

323322
r = None
@@ -347,8 +346,8 @@ def _obtain_region(self, a, offset, size, flags, is_recursive):
347346
self._memory_size += r.size()
348347
# END handle array
349348

350-
assert a.includes_ofs(offset)
351-
assert a.includes_ofs(offset + size-1)
349+
assert r.includes_ofs(offset)
350+
assert r.includes_ofs(offset + size-1)
352351
return r
353352

354353
#}END internal methods
@@ -362,6 +361,8 @@ def make_cursor(self, path_or_fd):
362361
your existing file descriptor, but keep in mind that new windows can only
363362
be mapped as long as it stays valid. This is why the using actual file paths
364363
are preferred unless you plan to keep the file descriptor open.
364+
:note: file descriptors are problematic as they are not necessarily unique, as two
365+
different files opened and closed in succession might have the same file descriptor id.
365366
:note: Using file descriptors directly is faster once new windows are mapped as it
366367
prevents the file to be opened again just for the purpose of mapping it."""
367368
regions = self._fdict.get(path_or_fd)
@@ -448,41 +449,10 @@ class SlidingWindowMapManager(StaticWindowMapManager):
448449

449450
__slots__ = tuple()
450451

451-
def __init__(self, window_size = 0, max_memory_size = 0, max_open_handles = sys.maxint):
452-
"""Adjusts the default window size to 0"""
452+
def __init__(self, window_size = -1, max_memory_size = 0, max_open_handles = sys.maxint):
453+
"""Adjusts the default window size to -1"""
453454
super(SlidingWindowMapManager, self).__init__(window_size, max_memory_size, max_open_handles)
454455

455-
def _collect_lru_region(self, size):
456-
num_found = 0
457-
while (size == 0) or (self._memory_size + size > self._max_memory_size):
458-
lru_region = None
459-
lru_list = None
460-
for regions in self._fdict.itervalues():
461-
for region in regions:
462-
# check client count - consider that we keep one reference ourselves !
463-
if (region.client_count()-2 == 0 and
464-
(lru_region is None or region._uc < lru_region._uc)):
465-
lru_region = region
466-
lru_list = regions
467-
# END update lru_region
468-
#END for each region
469-
#END for each regions list
470-
471-
if lru_region is None:
472-
if num_found == 0 and size != 0:
473-
raise RegionCollectionError("Didn't find any region to free")
474-
#END raise if necessary
475-
break
476-
#END handle region not found
477-
478-
num_found += 1
479-
del(lru_list[lru_list.index(lru_region)])
480-
self._memory_size -= lru_region.size()
481-
self._handle_count -= 1
482-
#END while there is more memory to free
483-
484-
return num_found
485-
486456
def _obtain_region(self, a, offset, size, flags, is_recursive):
487457
# bisect to find an existing region. The c++ implementation cannot
488458
# do that as it uses a linked list for regions.

‎smmap/test/test_mman.py‎

Lines changed: 39 additions & 32 deletions
Original file line numberDiff line numberDiff line change
@@ -46,40 +46,47 @@ def test_cursor(self):
4646
WindowCursor(man)._destroy()
4747

4848
def test_memory_manager(self):
49-
man = SlidingWindowMapManager()
50-
assert man.num_file_handles() == 0
51-
assert man.num_open_files() == 0
52-
assert man.window_size() > 0
53-
assert man.mapped_memory_size() == 0
54-
assert man.max_mapped_memory_size() > 0
55-
56-
# collection doesn't raise in 'any' mode
57-
man._collect_lru_region(0)
58-
# doesn't raise if we are within the limit
59-
man._collect_lru_region(10)
60-
# raises outside of limit
61-
self.failUnlessRaises(RegionCollectionError, man._collect_lru_region, sys.maxint)
49+
slide_man = SlidingWindowMapManager()
50+
static_man = StaticWindowMapManager()
6251

63-
# use a region, verify most basic functionality
64-
fc = FileCreator(self.k_window_test_size, "manager_test")
65-
fd = os.open(fc.path, os.O_RDONLY)
66-
for item in (fc.path, fd):
67-
c = man.make_cursor(item)
68-
assert c.path_or_fd() is item
69-
assert c.use_region(10, 10).is_valid()
70-
assert c.ofs_begin() == 10
71-
assert c.size() == 10
72-
assert c.buffer()[:] == open(fc.path, 'rb').read(20)[10:]
52+
for man in (static_man, slide_man):
53+
assert man.num_file_handles() == 0
54+
assert man.num_open_files() == 0
55+
winsize_cmp_val = 0
56+
if isinstance(man, StaticWindowMapManager):
57+
winsize_cmp_val = -1
58+
#END handle window size
59+
assert man.window_size() > winsize_cmp_val
60+
assert man.mapped_memory_size() == 0
61+
assert man.max_mapped_memory_size() > 0
62+
63+
# collection doesn't raise in 'any' mode
64+
man._collect_lru_region(0)
65+
# doesn't raise if we are within the limit
66+
man._collect_lru_region(10)
67+
# raises outside of limit
68+
self.failUnlessRaises(RegionCollectionError, man._collect_lru_region, sys.maxint)
69+
70+
# use a region, verify most basic functionality
71+
fc = FileCreator(self.k_window_test_size, "manager_test")
72+
fd = os.open(fc.path, os.O_RDONLY)
73+
for item in (fc.path, fd):
74+
c = man.make_cursor(item)
75+
assert c.path_or_fd() is item
76+
assert c.use_region(10, 10).is_valid()
77+
assert c.ofs_begin() == 10
78+
assert c.size() == 10
79+
assert c.buffer()[:] == open(fc.path, 'rb').read(20)[10:]
80+
81+
if isinstance(item, int):
82+
self.failUnlessRaises(ValueError, c.path)
83+
else:
84+
self.failUnlessRaises(ValueError, c.fd)
85+
#END handle value error
86+
#END for each input
87+
os.close(fd)
88+
# END for each manager type
7389

74-
if isinstance(item, int):
75-
self.failUnlessRaises(ValueError, c.path)
76-
else:
77-
self.failUnlessRaises(ValueError, c.fd)
78-
#END handle value error
79-
#END for each input
80-
81-
os.close(fd)
82-
8390
def test_memman_operation(self):
8491
# test more access, force it to actually unmap regions
8592
fc = FileCreator(self.k_window_test_size, "manager_operation_test")

0 commit comments

Comments
 (0)