"Programming Language :: Python :: 3.4",

],

long_description=long_description,

)

from .lib import TestBase, FileCreator

from smmap.mman import SlidingWindowMapManager, StaticWindowMapManager

static_man = StaticWindowMapManager()

class TestBuf(TestBase):

def test_basics(self):

fc = FileCreator(self.k_window_test_size, "buffer_test")

# invalid paths fail upon construction

c = man_optimal.make_cursor(fc.path)

self.assertRaises(ValueError, SlidingWindowMapBuffer, type(c)()) # invalid cursor

self.assertRaises(ValueError, SlidingWindowMapBuffer, c, fc.size) # offset too large

buf = SlidingWindowMapBuffer() # can create uninitailized buffers

assert buf.cursor() is None

# can call end access any time

buf.end_access()

buf.end_access()

assert len(buf) == 0

# begin access can revive it, if the offset is suitable

offset = 100

assert buf.begin_access(c, fc.size) == False

assert buf.begin_access(c, offset) == True

assert len(buf) == fc.size - offset

assert buf.cursor().is_valid()

# empty begin access keeps it valid on the same path, but alters the offset

assert buf.begin_access() == True

assert len(buf) == fc.size

assert buf.cursor().is_valid()

# simple access

assert data[offset] == buf[0]

assert data[offset:offset*2] == buf[0:offset]

# negative indices, partial slices

assert buf[-1] == buf[len(buf)-1]

assert buf[-10:] == buf[len(buf)-10:len(buf)]

# end access makes its cursor invalid

buf.end_access()

assert not buf.cursor().is_valid()

assert buf.cursor().is_associated() # but it remains associated

# an empty begin access fixes it up again

assert buf.begin_access() == True and buf.cursor().is_valid()

del(buf) # ends access automatically

del(c)

assert man_optimal.num_file_handles() == 1

# PERFORMANCE

# exagerate the manager's overhead, but measure the buffer overhead

# will produce small mappings only !

max_num_accesses = 100

fd = os.open(fc.path, os.O_RDONLY)

for item in (fc.path, fd):

(man_worst_case, 'worst case'),

(static_man, 'static optimal')):

buf = SlidingWindowMapBuffer(manager.make_cursor(item))

num_accesses_left = max_num_accesses

num_bytes = 0

fsize = fc.size

st = time()

buf.begin_access()

while num_accesses_left:

num_bytes += 1

#END handle mode

# END handle num accesses

buf.end_access()

assert manager.num_file_handles()

assert manager.collect()

assert manager.num_file_handles() == 0

elapsed = max(time() - st, 0.001) # prevent zero division errors on windows

mb = float(1000*1000)

mode_str = (access_mode and "slice") or "single byte"

# END handle access mode

# END for each manager

# END for each input

from .lib import TestBase, FileCreator

from smmap.mman import *

from copy import copy

class TestMMan(TestBase):

def test_cursor(self):

fc = FileCreator(self.k_window_test_size, "cursor_test")

man = SlidingWindowMapManager()

ci = WindowCursor(man) # invalid cursor

assert not ci.is_valid()

assert not ci.is_associated()

assert ci.size() == 0 # this is cached, so we can query it in invalid state

cv = man.make_cursor(fc.path)

assert not cv.is_valid() # no region mapped yet

assert cv.is_associated()# but it know where to map it from

assert cv.file_size() == fc.size

assert cv.path() == fc.path

# copy module

cio = copy(cv)

assert not cio.is_valid() and cio.is_associated()

# assign method

assert not ci.is_associated()

ci.assign(cv)

assert not ci.is_valid() and ci.is_associated()

# unuse non-existing region is fine

cv.unuse_region()

cv.unuse_region()

# destruction is fine (even multiple times)

cv._destroy()

WindowCursor(man)._destroy()

def test_memory_manager(self):

slide_man = SlidingWindowMapManager()

static_man = StaticWindowMapManager()

for man in (static_man, slide_man):

assert man.num_file_handles() == 0

assert man.num_open_files() == 0

assert man.window_size() > winsize_cmp_val

assert man.mapped_memory_size() == 0

assert man.max_mapped_memory_size() > 0

# collection doesn't raise in 'any' mode

man._collect_lru_region(0)

# doesn't raise if we are within the limit

man._collect_lru_region(10)

assert man._collect_lru_region(sys.maxsize) == 0

# use a region, verify most basic functionality

fc = FileCreator(self.k_window_test_size, "manager_test")

fd = os.open(fc.path, os.O_RDONLY)

assert c.use_region(10, 10).is_valid()

assert c.ofs_begin() == 10

assert c.size() == 10

if isinstance(item, int):

self.assertRaises(ValueError, c.path)

else:

#END for each input

os.close(fd)

# END for each manager type

def test_memman_operation(self):

# test more access, force it to actually unmap regions

fc = FileCreator(self.k_window_test_size, "manager_operation_test")

fd = os.open(fc.path, os.O_RDONLY)

max_num_handles = 15

for mtype, args in ( (StaticWindowMapManager, (0, fc.size // 3, max_num_handles)),

(SlidingWindowMapManager, (fc.size // 100, fc.size // 3, max_num_handles)),):

for item in (fc.path, fd):

assert len(data) == fc.size

# small windows, a reasonable max memory. Not too many regions at once

man = mtype(window_size=args[0], max_memory_size=args[1], max_open_handles=args[2])

c = man.make_cursor(item)

# still empty (more about that is tested in test_memory_manager()

assert man.num_open_files() == 0

assert man.mapped_memory_size() == 0

base_offset = 5000

# window size is 0 for static managers, hence size will be 0. We take that into consideration

size = man.window_size() // 2

assert c.use_region(base_offset, size).is_valid()

rr = c.region_ref()

assert rr().client_count() == 2 # the manager and the cursor and us

assert man.num_open_files() == 1

assert man.num_file_handles() == 1

assert man.mapped_memory_size() == rr().size()

#assert c.size() == size # the cursor may overallocate in its static version

assert c.ofs_begin() == base_offset

assert rr().ofs_begin() == 0 # it was aligned and expanded

else:

assert rr().size() == fc.size

#END ignore static managers which dont use windows and are aligned to file boundaries

# obtain second window, which spans the first part of the file - it is a still the same window

nsize = (size or fc.size) - 10

assert c.use_region(0, nsize).is_valid()

assert c.size() == nsize

assert c.ofs_begin() == 0

assert c.buffer()[:] == data[:nsize]

# map some part at the end, our requested size cannot be kept

overshoot = 4000

base_offset = fc.size - (size or c.size()) + overshoot

assert rr().ofs_begin() < c.ofs_begin() # it should have extended itself to the left

assert rr().ofs_end() <= fc.size # it cannot be larger than the file

assert c.buffer()[:] == data[base_offset:base_offset+(size or c.size())]

# unising a region makes the cursor invalid

c.unuse_region()

assert not c.is_valid()

if man.window_size():

# remove mapped regions if we have to

assert man.num_file_handles() == 2

#END ignore this for static managers

# iterate through the windows, verify data contents

# this will trigger map collection after a while

max_random_accesses = 5000

num_random_accesses = max_random_accesses

memory_read = 0

st = time()

# cache everything to get some more performance

includes_ofs = c.includes_ofs

max_mapped_memory_size = man.max_mapped_memory_size()

while num_random_accesses:

num_random_accesses -= 1

base_offset = randint(0, fc.size - 1)

# precondition

if man.window_size():

assert max_mapped_memory_size >= mapped_memory_size()

csize = c.size()

assert c.buffer()[:] == data[base_offset:base_offset+csize]

memory_read += csize

assert includes_ofs(base_offset)

assert includes_ofs(base_offset+csize-1)

assert not includes_ofs(base_offset+csize)

# END while we should do an access

elapsed = max(time() - st, 0.001) # prevent zero divison errors on windows

mb = float(1000 * 1000)

# an offset as large as the size doesn't work !

assert not c.use_region(fc.size, size).is_valid()

# collection - it should be able to collect all

assert man.num_file_handles()

assert man.collect()