Hi, thanks a lot for adding Kilosort 4 to the pipeline so quickly! I get an "numpy.core._exceptions._ArrayMemoryError" error when I run spike sorting with it. Has someone encountered and solved this error before?

Thanks in advance!

import spikeinterface.preprocessing as spre # spikeinterface 0.100.5

# load recording extractor
Wired10mInt16 = si.load_extractor(RECORDING_EXTRACTOR_PATH)
print(Wired10mInt16)

# convert to int16
Wired10mInt16 = spre.astype(Wired10mInt16, "int16")

# sort
sorting_KS4 = ss.run_sorter(sorter_name='kilosort4',
                            recording=Wired10mInt16,
                            remove_existing_folder=True,
                            output_folder=SORTING_PATH,
                            verbose=True)

Output:

BinaryFolderRecording: 384 channels - 40.0kHz - 1 segments - 24,000,000 samples
600.00s (10.00 minutes) - int16 dtype - 17.17 GiB

write_binary_recording: 0%| | 0/1 [00:00<?, ?it/s]
write_binary_recording: 100%|██████████| 1/1 [12:28<00:00, 748.47s/it]
write_binary_recording: 100%|██████████| 1/1 [12:28<00:00, 748.47s/it]
/gpfs/bbp.cscs.ch/project/proj85/scratch/laquitai/4_preprint_2023/envs/spikinterf0_100_5/lib/python3.9/site-packages/kilosort/io.py:497: UserWarning: The given NumPy array is not writable, and PyTorch does not support non-writable tensors. This means writing to this tensor will result in undefined behavior. You may want to copy the array to protect its data or make it writable before converting it to a tensor. This type of warning will be suppressed for the rest of this program. (Triggered internally at ../torch/csrc/utils/tensor_numpy.cpp:206.)
X[:, self.nt : self.nt+nsamp] = torch.from_numpy(data).to(self.device).float()
Traceback (most recent call last):
File "/gpfs/bbp.cscs.ch/home/laquitai/spack_install/environments/spack_python3_9/.spack-env/view/lib/python3.9/runpy.py", line 197, in _run_module_as_main
return _run_code(code, main_globals, None,
File "/gpfs/bbp.cscs.ch/home/laquitai/spack_install/environments/spack_python3_9/.spack-env/view/lib/python3.9/runpy.py", line 87, in _run_code
exec(code, run_globals)
File "/gpfs/bbp.cscs.ch/project/proj85/home/laquitai/preprint_2023/src/pipes/sorting/marques_silico/10m/buttw/ks4/sort_noise_ftd_gain_ftd.py", line 130, in
sorting_KS4 = ss.run_sorter(sorter_name='kilosort4',
File "/gpfs/bbp.cscs.ch/project/proj85/scratch/laquitai/4_preprint_2023/envs/spikinterf0_100_5/lib/python3.9/site-packages/spikeinterface/sorters/runsorter.py", line 175, in run_sorter
return run_sorter_local(**common_kwargs)
File "/gpfs/bbp.cscs.ch/project/proj85/scratch/laquitai/4_preprint_2023/envs/spikinterf0_100_5/lib/python3.9/site-packages/spikeinterface/sorters/runsorter.py", line 225, in run_sorter_local
SorterClass.run_from_folder(output_folder, raise_error, verbose)
File "/gpfs/bbp.cscs.ch/project/proj85/scratch/laquitai/4_preprint_2023/envs/spikinterf0_100_5/lib/python3.9/site-packages/spikeinterface/sorters/basesorter.py", line 293, in run_from_folder
raise SpikeSortingError(
spikeinterface.sorters.utils.misc.SpikeSortingError: Spike sorting error trace:
Traceback (most recent call last):
File "/gpfs/bbp.cscs.ch/project/proj85/scratch/laquitai/4_preprint_2023/envs/spikinterf0_100_5/lib/python3.9/site-packages/spikeinterface/sorters/basesorter.py", line 258, in run_from_folder
SorterClass._run_from_folder(sorter_output_folder, sorter_params, verbose)
File "/gpfs/bbp.cscs.ch/project/proj85/scratch/laquitai/4_preprint_2023/envs/spikinterf0_100_5/lib/python3.9/site-packages/spikeinterface/sorters/external/kilosort4.py", line 260, in _run_from_folder
st, tF, _, _ = detect_spikes(ops, device, bfile, tic0=tic0, progress_bar=progress_bar)
File "/gpfs/bbp.cscs.ch/project/proj85/scratch/laquitai/4_preprint_2023/envs/spikinterf0_100_5/lib/python3.9/site-packages/kilosort/run_kilosort.py", line 392, in detect_spikes
st0, tF, ops = spikedetect.run(ops, bfile, device=device, progress_bar=progress_bar)
File "/gpfs/bbp.cscs.ch/project/proj85/scratch/laquitai/4_preprint_2023/envs/spikinterf0_100_5/lib/python3.9/site-packages/kilosort/spikedetect.py", line 215, in run
iC2, ds2 = nearest_chans(ys, ys, xs, xs, nC2, device=device)
File "/gpfs/bbp.cscs.ch/project/proj85/scratch/laquitai/4_preprint_2023/envs/spikinterf0_100_5/lib/python3.9/site-packages/kilosort/spikedetect.py", line 171, in nearest_chans
ds = (ys - yc[:,np.newaxis])**2 + (xs - xc[:,np.newaxis])**2
numpy.core._exceptions._ArrayMemoryError: Unable to allocate 117. TiB for an array with shape (4002249, 4002249) and data type float64