Thanks to visit codestin.com
Credit goes to github.com

Skip to content

MemoryError when execute feature export #36

@liuyezhe

Description

@liuyezhe

I used nellie to analyze a 2048x2048, 16bit data, and encountered a MemoryError when performing the feature export operation. The specific prompt is as follows:

---------------------------------------------------------------------------
MemoryError                               Traceback (most recent call last)
File D:\Anaconda3\envs\Nellie\lib\site-packages\superqt\utils\_qthreading.py:613, in create_worker.<locals>.reraise(e=MemoryError((95491, 95491, 2), dtype('float64')))
    612 def reraise(e):
--> 613     raise e
        e = MemoryError((95491, 95491, 2), dtype('float64'))

File D:\Anaconda3\envs\Nellie\lib\site-packages\superqt\utils\_qthreading.py:175, in WorkerBase.run(self=<napari._qt.qthreading.FunctionWorker object>)
    173     warnings.filterwarnings("always")
    174     warnings.showwarning = lambda *w: self.warned.emit(w)
--> 175     result = self.work()
        self = <napari._qt.qthreading.FunctionWorker object at 0x000001566E0EE940>
    176 if isinstance(result, Exception):
    177     if isinstance(result, RuntimeError):
    178         # The Worker object has likely been deleted.
    179         # A deleted wrapped C/C++ object may result in a runtime
    180         # error that will cause segfault if we try to do much other
    181         # than simply notify the user.

File D:\Anaconda3\envs\Nellie\lib\site-packages\superqt\utils\_qthreading.py:354, in FunctionWorker.work(self=<napari._qt.qthreading.FunctionWorker object>)
    353 def work(self) -> _R:
--> 354     return self._func(*self._args, **self._kwargs)
        self._func = <function NellieProcessor._run_feature_export at 0x000001566D723F70>
        self = <napari._qt.qthreading.FunctionWorker object at 0x000001566E0EE940>
        self._args = (<nellie_napari.nellie_processor.NellieProcessor object at 0x000001566E0FA5E0>,)
        self._kwargs = {}

File D:\Anaconda3\envs\Nellie\lib\site-packages\nellie_napari\nellie_processor.py:452, in NellieProcessor._run_feature_export(self=<nellie_napari.nellie_processor.NellieProcessor object>)
    448 self.current_im_info = im_info
    449 hierarchy = Hierarchy(im_info=self.current_im_info,
    450                       skip_nodes=not bool(self.nellie.settings.analyze_node_level.isChecked()),
    451                       viewer=self.viewer)
--> 452 hierarchy.run()
        hierarchy = <nellie.feature_extraction.hierarchical.Hierarchy object at 0x00000156728D5250>
    453 if self.nellie.settings.remove_intermediates_checkbox.isChecked():
    454     try:

File D:\Anaconda3\envs\Nellie\lib\site-packages\nellie\feature_extraction\hierarchical.py:322, in Hierarchy.run(self=<nellie.feature_extraction.hierarchical.Hierarchy object>)
    320 self._get_t()
    321 self._allocate_memory()
--> 322 self._get_hierarchies()
        self = <nellie.feature_extraction.hierarchical.Hierarchy object at 0x00000156728D5250>
    323 self._save_dfs()
    324 if self.viewer is not None:

File D:\Anaconda3\envs\Nellie\lib\site-packages\nellie\feature_extraction\hierarchical.py:172, in Hierarchy._get_hierarchies(self=<nellie.feature_extraction.hierarchical.Hierarchy object>)
    170 logger.info("Running branch analysis")
    171 start = time.time()
--> 172 self.branches.run()
        self.branches = <nellie.feature_extraction.hierarchical.Branches object at 0x00000156622A8460>
        self = <nellie.feature_extraction.hierarchical.Hierarchy object at 0x00000156728D5250>
    173 end = time.time()
    174 b_time = end - start

File D:\Anaconda3\envs\Nellie\lib\site-packages\nellie\feature_extraction\hierarchical.py:1668, in Branches.run(self=<nellie.feature_extraction.hierarchical.Branches object>)
   1666 if self.hierarchy.viewer is not None:
   1667     self.hierarchy.viewer.status = f'Extracting branch features. Frame: {t + 1} of {self.hierarchy.num_t}.'
-> 1668 self._run_frame(t)
        t = 0
        self = <nellie.feature_extraction.hierarchical.Branches object at 0x00000156622A8460>

File D:\Anaconda3\envs\Nellie\lib\site-packages\nellie\feature_extraction\hierarchical.py:1658, in Branches._run_frame(self=<nellie.feature_extraction.hierarchical.Branches object>, t=0)
   1655 self.image_name.append(im_name)
   1657 self._get_aggregate_stats(t)
-> 1658 self._get_branch_stats(t)
        t = 0
        self = <nellie.feature_extraction.hierarchical.Branches object at 0x00000156622A8460>

File D:\Anaconda3\envs\Nellie\lib\site-packages\nellie\feature_extraction\hierarchical.py:1492, in Branches._get_branch_stats(self=<nellie.feature_extraction.hierarchical.Branches object>, t=0)
   1490 branch_idx_array_1 = np.array(self.branch_idxs[t])
   1491 branch_idx_array_2 = np.array(self.branch_idxs[t])[:, None, :]
-> 1492 dist = np.linalg.norm(branch_idx_array_1 - branch_idx_array_2, axis=-1)
        branch_idx_array_1 = array([[   0, 1602],
       [   0, 1603],
       ...,
       [2046, 1985],
       [2046, 1997]], dtype=int64)
        branch_idx_array_2 = array([[[   0, 1602]],

       [[   0, 1603]],

       ...,

       [[2046, 1985]],

       [[2046, 1997]]], dtype=int64)
        np.linalg.norm = <function norm at 0x0000015629F89330>
        np = <module 'numpy' from 'D:\\Anaconda3\\envs\\Nellie\\lib\\site-packages\\numpy\\__init__.py'>
        np.linalg = <module 'numpy.linalg' from 'D:\\Anaconda3\\envs\\Nellie\\lib\\site-packages\\numpy\\linalg\\__init__.py'>
   1493 dist[dist >= 2] = 0  # remove any distances greater than adjacent pixel
   1494 neighbors = np.sum(dist > 0, axis=1)

File D:\Anaconda3\envs\Nellie\lib\site-packages\numpy\linalg\linalg.py:2537, in norm(x=array([[[    0,     0],
        [    0,     1],
...0,   -12],
        [    0,     0]]], dtype=int64), ord=None, axis=-1, keepdims=False)
   2534 x = asarray(x)
   2536 if not issubclass(x.dtype.type, (inexact, object_)):
-> 2537     x = x.astype(float)
        x = array([[[    0,     0],
        [    0,     1],
        ...,
        [ 2046,   383],
        [ 2046,   395]],

       [[    0,    -1],
        [    0,     0],
        ...,
        [ 2046,   382],
        [ 2046,   394]],

       ...,

       [[-2046,  -383],
        [-2046,  -382],
        ...,
        [    0,     0],
        [    0,    12]],

       [[-2046,  -395],
        [-2046,  -394],
        ...,
        [    0,   -12],
        [    0,     0]]], dtype=int64)
   2539 # Immediately handle some default, simple, fast, and common cases.
   2540 if axis is None:

MemoryError: Unable to allocate 136. GiB for an array with shape (95491, 95491, 2) and data type float64

Is it because too many branches were identified that led to the failure of the calculation? Are there any parameters I can adjust to modify the number of identified branches? Or are there any parameters that can be adjusted for the preprocess? (because I think the processing is not quite ideal) Or can I choose to only perform organelle-level analysis?

Metadata

Metadata

Assignees

Projects

No projects

Milestone

No milestone

Relationships

None yet

Development

No branches or pull requests

Issue actions