-
Notifications
You must be signed in to change notification settings - Fork 12
Open
Labels
Description
I used nellie to analyze a 2048x2048, 16bit data, and encountered a MemoryError when performing the feature export operation. The specific prompt is as follows:
---------------------------------------------------------------------------
MemoryError Traceback (most recent call last)
File D:\Anaconda3\envs\Nellie\lib\site-packages\superqt\utils\_qthreading.py:613, in create_worker.<locals>.reraise(e=MemoryError((95491, 95491, 2), dtype('float64')))
612 def reraise(e):
--> 613 raise e
e = MemoryError((95491, 95491, 2), dtype('float64'))
File D:\Anaconda3\envs\Nellie\lib\site-packages\superqt\utils\_qthreading.py:175, in WorkerBase.run(self=<napari._qt.qthreading.FunctionWorker object>)
173 warnings.filterwarnings("always")
174 warnings.showwarning = lambda *w: self.warned.emit(w)
--> 175 result = self.work()
self = <napari._qt.qthreading.FunctionWorker object at 0x000001566E0EE940>
176 if isinstance(result, Exception):
177 if isinstance(result, RuntimeError):
178 # The Worker object has likely been deleted.
179 # A deleted wrapped C/C++ object may result in a runtime
180 # error that will cause segfault if we try to do much other
181 # than simply notify the user.
File D:\Anaconda3\envs\Nellie\lib\site-packages\superqt\utils\_qthreading.py:354, in FunctionWorker.work(self=<napari._qt.qthreading.FunctionWorker object>)
353 def work(self) -> _R:
--> 354 return self._func(*self._args, **self._kwargs)
self._func = <function NellieProcessor._run_feature_export at 0x000001566D723F70>
self = <napari._qt.qthreading.FunctionWorker object at 0x000001566E0EE940>
self._args = (<nellie_napari.nellie_processor.NellieProcessor object at 0x000001566E0FA5E0>,)
self._kwargs = {}
File D:\Anaconda3\envs\Nellie\lib\site-packages\nellie_napari\nellie_processor.py:452, in NellieProcessor._run_feature_export(self=<nellie_napari.nellie_processor.NellieProcessor object>)
448 self.current_im_info = im_info
449 hierarchy = Hierarchy(im_info=self.current_im_info,
450 skip_nodes=not bool(self.nellie.settings.analyze_node_level.isChecked()),
451 viewer=self.viewer)
--> 452 hierarchy.run()
hierarchy = <nellie.feature_extraction.hierarchical.Hierarchy object at 0x00000156728D5250>
453 if self.nellie.settings.remove_intermediates_checkbox.isChecked():
454 try:
File D:\Anaconda3\envs\Nellie\lib\site-packages\nellie\feature_extraction\hierarchical.py:322, in Hierarchy.run(self=<nellie.feature_extraction.hierarchical.Hierarchy object>)
320 self._get_t()
321 self._allocate_memory()
--> 322 self._get_hierarchies()
self = <nellie.feature_extraction.hierarchical.Hierarchy object at 0x00000156728D5250>
323 self._save_dfs()
324 if self.viewer is not None:
File D:\Anaconda3\envs\Nellie\lib\site-packages\nellie\feature_extraction\hierarchical.py:172, in Hierarchy._get_hierarchies(self=<nellie.feature_extraction.hierarchical.Hierarchy object>)
170 logger.info("Running branch analysis")
171 start = time.time()
--> 172 self.branches.run()
self.branches = <nellie.feature_extraction.hierarchical.Branches object at 0x00000156622A8460>
self = <nellie.feature_extraction.hierarchical.Hierarchy object at 0x00000156728D5250>
173 end = time.time()
174 b_time = end - start
File D:\Anaconda3\envs\Nellie\lib\site-packages\nellie\feature_extraction\hierarchical.py:1668, in Branches.run(self=<nellie.feature_extraction.hierarchical.Branches object>)
1666 if self.hierarchy.viewer is not None:
1667 self.hierarchy.viewer.status = f'Extracting branch features. Frame: {t + 1} of {self.hierarchy.num_t}.'
-> 1668 self._run_frame(t)
t = 0
self = <nellie.feature_extraction.hierarchical.Branches object at 0x00000156622A8460>
File D:\Anaconda3\envs\Nellie\lib\site-packages\nellie\feature_extraction\hierarchical.py:1658, in Branches._run_frame(self=<nellie.feature_extraction.hierarchical.Branches object>, t=0)
1655 self.image_name.append(im_name)
1657 self._get_aggregate_stats(t)
-> 1658 self._get_branch_stats(t)
t = 0
self = <nellie.feature_extraction.hierarchical.Branches object at 0x00000156622A8460>
File D:\Anaconda3\envs\Nellie\lib\site-packages\nellie\feature_extraction\hierarchical.py:1492, in Branches._get_branch_stats(self=<nellie.feature_extraction.hierarchical.Branches object>, t=0)
1490 branch_idx_array_1 = np.array(self.branch_idxs[t])
1491 branch_idx_array_2 = np.array(self.branch_idxs[t])[:, None, :]
-> 1492 dist = np.linalg.norm(branch_idx_array_1 - branch_idx_array_2, axis=-1)
branch_idx_array_1 = array([[ 0, 1602],
[ 0, 1603],
...,
[2046, 1985],
[2046, 1997]], dtype=int64)
branch_idx_array_2 = array([[[ 0, 1602]],
[[ 0, 1603]],
...,
[[2046, 1985]],
[[2046, 1997]]], dtype=int64)
np.linalg.norm = <function norm at 0x0000015629F89330>
np = <module 'numpy' from 'D:\\Anaconda3\\envs\\Nellie\\lib\\site-packages\\numpy\\__init__.py'>
np.linalg = <module 'numpy.linalg' from 'D:\\Anaconda3\\envs\\Nellie\\lib\\site-packages\\numpy\\linalg\\__init__.py'>
1493 dist[dist >= 2] = 0 # remove any distances greater than adjacent pixel
1494 neighbors = np.sum(dist > 0, axis=1)
File D:\Anaconda3\envs\Nellie\lib\site-packages\numpy\linalg\linalg.py:2537, in norm(x=array([[[ 0, 0],
[ 0, 1],
...0, -12],
[ 0, 0]]], dtype=int64), ord=None, axis=-1, keepdims=False)
2534 x = asarray(x)
2536 if not issubclass(x.dtype.type, (inexact, object_)):
-> 2537 x = x.astype(float)
x = array([[[ 0, 0],
[ 0, 1],
...,
[ 2046, 383],
[ 2046, 395]],
[[ 0, -1],
[ 0, 0],
...,
[ 2046, 382],
[ 2046, 394]],
...,
[[-2046, -383],
[-2046, -382],
...,
[ 0, 0],
[ 0, 12]],
[[-2046, -395],
[-2046, -394],
...,
[ 0, -12],
[ 0, 0]]], dtype=int64)
2539 # Immediately handle some default, simple, fast, and common cases.
2540 if axis is None:
MemoryError: Unable to allocate 136. GiB for an array with shape (95491, 95491, 2) and data type float64Is it because too many branches were identified that led to the failure of the calculation? Are there any parameters I can adjust to modify the number of identified branches? Or are there any parameters that can be adjusted for the preprocess? (because I think the processing is not quite ideal) Or can I choose to only perform organelle-level analysis?