Thanks to visit codestin.com
Credit goes to github.com

Skip to content

Commit bc78951

Browse files
ByronByron
committed
moved code from cursor into manager, as it belongs there. This is a design error inherited from c++, but actually it makes overrides a bit harder, or lets say, less native, as the sliding mechanics where implemented in a class which is just the handle to a memory map in the end, which doesn't have to care about its allocation
1 parent 9dc4a8d commit bc78951

3 files changed

Lines changed: 122 additions & 120 deletions

File tree

‎smmap/buf.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
"""Module with a simple buffer implementation using the memory manager"""
2-
from mman import SlidingCursor
2+
from mman import MemoryCursor
33

44
import sys
55

‎smmap/mman.py‎

Lines changed: 118 additions & 116 deletions
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,7 @@
1515

1616
#}END utilities
1717

18-
class SlidingCursor(object):
18+
class MemoryCursor(object):
1919
"""Pointer into the mapped region of the memory manager, keeping the current window
2020
alive until it is destroyed.
2121
@@ -28,11 +28,6 @@ class SlidingCursor(object):
2828
'_size' # maximum size we should provide
2929
)
3030

31-
#{ Configuration
32-
MapWindowCls = MapWindow
33-
MapRegionCls = MapRegion
34-
#} END configuration
35-
3631
def __init__(self, manager = None, regions = None):
3732
self._manager = manager
3833
self._rlist = regions
@@ -82,7 +77,7 @@ def assign(self, rhs):
8277
self._destroy()
8378
self._copy_from(rhs)
8479

85-
def use_region(self, offset, size, flags = 0, _is_recursive=False):
80+
def use_region(self, offset, size, flags = 0):
8681
"""Assure we point to a window which allows access to the given offset into the file
8782
:param offset: absolute offset in bytes into the file
8883
:param size: amount of bytes to map
@@ -104,115 +99,14 @@ def use_region(self, offset, size, flags = 0, _is_recursive=False):
10499
# END handle existing region
105100
# END check existing region
106101

102+
# offset too large ?
103+
if offset >= self._rlist.file_size():
104+
return self
105+
#END handle offset
106+
107107
if need_region:
108-
window_size = man._window_size
109-
110-
# abort on offsets beyond our mapped file's size - currently we are invalid
111-
if offset >= self.file_size():
112-
return self
113-
# END handle offset too large
114-
115-
# bisect to find an existing region. The c++ implementation cannot
116-
# do that as it uses a linked list for regions.
117-
existing_region = None
118-
a = self._rlist
119-
lo = 0
120-
hi = len(a)
121-
while lo < hi:
122-
mid = (lo+hi)//2
123-
ofs = a[mid]._b
124-
if ofs <= offset:
125-
if a[mid].includes_ofs(offset):
126-
existing_region = a[mid]
127-
break
128-
#END have region
129-
lo = mid+1
130-
else:
131-
hi = mid
132-
#END handle position
133-
#END while bisecting
134-
135-
if existing_region is None:
136-
left = self.MapWindowCls(0, 0)
137-
mid = self.MapWindowCls(offset, size)
138-
right = self.MapWindowCls(self.file_size(), 0)
139-
140-
# we want to honor the max memory size, and assure we have anough
141-
# memory available
142-
# Save calls !
143-
if self._manager._memory_size + window_size > self._manager._max_memory_size:
144-
man._collect_lru_region(window_size)
145-
#END handle collection
146-
147-
# we assume the list remains sorted by offset
148-
insert_pos = 0
149-
len_regions = len(a)
150-
if len_regions == 1:
151-
if a[0]._b <= offset:
152-
insert_pos = 1
153-
#END maintain sort
154-
else:
155-
# find insert position
156-
insert_pos = len_regions
157-
for i, region in enumerate(a):
158-
if region._b > offset:
159-
insert_pos = i
160-
break
161-
#END if insert position is correct
162-
#END for each region
163-
# END obtain insert pos
164-
165-
# adjust the actual offset and size values to create the largest
166-
# possible mapping
167-
if insert_pos == 0:
168-
if len_regions:
169-
right = self.MapWindowCls.from_region(a[insert_pos])
170-
#END adjust right side
171-
else:
172-
if insert_pos != len_regions:
173-
right = self.MapWindowCls.from_region(a[insert_pos])
174-
# END adjust right window
175-
left = self.MapWindowCls.from_region(a[insert_pos - 1])
176-
#END adjust surrounding windows
177-
178-
mid.extend_left_to(left, window_size)
179-
mid.extend_right_to(right, window_size)
180-
mid.align()
181-
182-
# it can happen that we align beyond the end of the file
183-
if mid.ofs_end() > right.ofs:
184-
mid.size = right.ofs - mid.ofs
185-
#END readjust size
186-
187-
# insert new region at the right offset to keep the order
188-
try:
189-
if man._handle_count >= man._max_handle_count:
190-
raise Exception
191-
#END assert own imposed max file handles
192-
self._region = self.MapRegionCls(a.path_or_fd(), mid.ofs, mid.size, flags)
193-
except Exception:
194-
# apparently we are out of system resources or hit a limit
195-
# As many more operations are likely to fail in that condition (
196-
# like reading a file from disk, etc) we free up as much as possible
197-
# As this invalidates our insert position, we have to recurse here
198-
# NOTE: The c++ version uses a linked list to curcumvent this, but
199-
# using that in python is probably too slow anyway
200-
if _is_recursive:
201-
# we already tried this, and still have no success in obtaining
202-
# a mapping. This is an exception, so we propagate it
203-
raise
204-
#END handle existing recursion
205-
man._collect_lru_region(0)
206-
return self.use_region(offset, size, flags, True)
207-
#END handle exceptions
208-
209-
man._handle_count += 1
210-
man._memory_size += self._region.size()
211-
a.insert(insert_pos, self._region)
212-
else:
213-
self._region = existing_region
214-
#END need region handling
215-
#END handle acquire region
108+
self._region = man._obtain_region(self._rlist, offset, size, flags, False)
109+
#END need region handling
216110

217111
self._region.increment_usage_count()
218112
self._ofs = offset - self._region._b
@@ -301,6 +195,7 @@ def fd(self):
301195
#} END interface
302196

303197

198+
304199
class SlidingWindowMapManager(object):
305200
"""Maintains a list of ranges of mapped memory regions in one or more files and allows to easily
306201
obtain additional regions assuring there is no overlap.
@@ -325,6 +220,8 @@ class SlidingWindowMapManager(object):
325220

326221
#{ Configuration
327222
MapRegionListCls = MapRegionList
223+
MapWindowCls = MapWindow
224+
MapRegionCls = MapRegion
328225
#} END configuration
329226

330227
_MB_in_bytes = 1024 * 1024
@@ -398,6 +295,111 @@ def _collect_lru_region(self, size):
398295

399296
return num_found
400297

298+
def _obtain_region(self, a, offset, size, flags, is_recursive):
299+
"""Utilty to create a new region - for more information on the parameters,
300+
see MapCursor.use_region.
301+
:param a: A regions (a)rray
302+
:return: The newly created region"""
303+
# bisect to find an existing region. The c++ implementation cannot
304+
# do that as it uses a linked list for regions.
305+
r = None
306+
lo = 0
307+
hi = len(a)
308+
while lo < hi:
309+
mid = (lo+hi)//2
310+
ofs = a[mid]._b
311+
if ofs <= offset:
312+
if a[mid].includes_ofs(offset):
313+
r = a[mid]
314+
break
315+
#END have region
316+
lo = mid+1
317+
else:
318+
hi = mid
319+
#END handle position
320+
#END while bisecting
321+
322+
if r is None:
323+
window_size = self._window_size
324+
left = self.MapWindowCls(0, 0)
325+
mid = self.MapWindowCls(offset, size)
326+
right = self.MapWindowCls(a.file_size(), 0)
327+
328+
# we want to honor the max memory size, and assure we have anough
329+
# memory available
330+
# Save calls !
331+
if self._memory_size + window_size > self._max_memory_size:
332+
self._collect_lru_region(window_size)
333+
#END handle collection
334+
335+
# we assume the list remains sorted by offset
336+
insert_pos = 0
337+
len_regions = len(a)
338+
if len_regions == 1:
339+
if a[0]._b <= offset:
340+
insert_pos = 1
341+
#END maintain sort
342+
else:
343+
# find insert position
344+
insert_pos = len_regions
345+
for i, region in enumerate(a):
346+
if region._b > offset:
347+
insert_pos = i
348+
break
349+
#END if insert position is correct
350+
#END for each region
351+
# END obtain insert pos
352+
353+
# adjust the actual offset and size values to create the largest
354+
# possible mapping
355+
if insert_pos == 0:
356+
if len_regions:
357+
right = self.MapWindowCls.from_region(a[insert_pos])
358+
#END adjust right side
359+
else:
360+
if insert_pos != len_regions:
361+
right = self.MapWindowCls.from_region(a[insert_pos])
362+
# END adjust right window
363+
left = self.MapWindowCls.from_region(a[insert_pos - 1])
364+
#END adjust surrounding windows
365+
366+
mid.extend_left_to(left, window_size)
367+
mid.extend_right_to(right, window_size)
368+
mid.align()
369+
370+
# it can happen that we align beyond the end of the file
371+
if mid.ofs_end() > right.ofs:
372+
mid.size = right.ofs - mid.ofs
373+
#END readjust size
374+
375+
# insert new region at the right offset to keep the order
376+
try:
377+
if self._handle_count >= self._max_handle_count:
378+
raise Exception
379+
#END assert own imposed max file handles
380+
r = self.MapRegionCls(a.path_or_fd(), mid.ofs, mid.size, flags)
381+
except Exception:
382+
# apparently we are out of system resources or hit a limit
383+
# As many more operations are likely to fail in that condition (
384+
# like reading a file from disk, etc) we free up as much as possible
385+
# As this invalidates our insert position, we have to recurse here
386+
# NOTE: The c++ version uses a linked list to curcumvent this, but
387+
# using that in python is probably too slow anyway
388+
if is_recursive:
389+
# we already tried this, and still have no success in obtaining
390+
# a mapping. This is an exception, so we propagate it
391+
raise
392+
#END handle existing recursion
393+
self._collect_lru_region(0)
394+
return self._obtain_region(a, offset, size, flags, True)
395+
#END handle exceptions
396+
397+
self._handle_count += 1
398+
self._memory_size += r.size()
399+
a.insert(insert_pos, r)
400+
# END create new region
401+
return r
402+
401403
#{ Interface
402404
def make_cursor(self, path_or_fd):
403405
""":return: a cursor pointing to the given path or file descriptor.
@@ -414,7 +416,7 @@ def make_cursor(self, path_or_fd):
414416
regions = self.MapRegionListCls(path_or_fd)
415417
self._fdict[path_or_fd] = regions
416418
# END obtain region for path
417-
return SlidingCursor(self, regions)
419+
return MemoryCursor(self, regions)
418420

419421
def collect(self):
420422
"""Collect all available free-to-collect mapped regions

‎smmap/test/test_mman.py‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
from lib import TestBase, FileCreator
22

33
from smmap.mman import *
4-
from smmap.mman import SlidingCursor
4+
from smmap.mman import MemoryCursor
55
from smmap.util import align_to_mmap
66
from smmap.exc import RegionCollectionError
77

@@ -17,7 +17,7 @@ def test_cursor(self):
1717
fc = FileCreator(self.k_window_test_size, "cursor_test")
1818

1919
man = SlidingWindowMapManager()
20-
ci = SlidingCursor(man) # invalid cursor
20+
ci = MemoryCursor(man) # invalid cursor
2121
assert not ci.is_valid()
2222
assert not ci.is_associated()
2323
assert ci.size() == 0 # this is cached, so we can query it in invalid state
@@ -43,7 +43,7 @@ def test_cursor(self):
4343

4444
# destruction is fine (even multiple times)
4545
cv._destroy()
46-
SlidingCursor(man)._destroy()
46+
MemoryCursor(man)._destroy()
4747

4848
def test_memory_manager(self):
4949
man = SlidingWindowMapManager()

0 commit comments

Comments
 (0)