From c309437914e488c381385d81174c71a0b0eae7b6 Mon Sep 17 00:00:00 2001 From: JulesLebert Date: Wed, 19 Apr 2023 15:01:19 +0100 Subject: [PATCH 1/5] load data with load insteand of memmap --- neo/rawio/tdtrawio.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/neo/rawio/tdtrawio.py b/neo/rawio/tdtrawio.py index f7f00df3a..3cea5e4ef 100644 --- a/neo/rawio/tdtrawio.py +++ b/neo/rawio/tdtrawio.py @@ -110,7 +110,7 @@ def _parse_header(self): for seg_index, segment_name in enumerate(segment_names): tev_filename = self._get_filestem(segment_name).with_suffix('.tev') if tev_filename.exists(): - tev_data = np.memmap(tev_filename, mode='r', offset=0, dtype='uint8') + tev_data = np.load(tev_filename, mode='r', offset=0, dtype='uint8') else: tev_data = None self._tev_datas.append(tev_data) @@ -266,7 +266,7 @@ def _parse_header(self): sev_filename = sev_filename[0] if (sev_filename is not None) and sev_filename.exists(): - data = np.memmap(sev_filename, mode='r', offset=0, dtype='uint8') + data = np.load(sev_filename, mode='r', offset=0, dtype='uint8') else: data = self._tev_datas[seg_index] assert data is not None, 'no TEV nor SEV' From 47049accfc5b5d87e4541e996c0cb16f391d75ff Mon Sep 17 00:00:00 2001 From: JulesLebert Date: Wed, 19 Apr 2023 15:38:02 +0100 Subject: [PATCH 2/5] stream name for spikeinterface --- neo/rawio/tdtrawio.py | 26 ++++++++++++++++---------- 1 file changed, 16 insertions(+), 10 deletions(-) diff --git a/neo/rawio/tdtrawio.py b/neo/rawio/tdtrawio.py index 3cea5e4ef..ae945d1e5 100644 --- a/neo/rawio/tdtrawio.py +++ b/neo/rawio/tdtrawio.py @@ -34,10 +34,9 @@ class TdtRawIO(BaseRawIO): - extensions = ['tbk', 'tdx', 'tev', 'tin', 'tnt', 'tsq', 'sev', 'txt'] rawmode = 'one-dir' - def __init__(self, dirname='', sortname=''): + def __init__(self, dirname='', sortname='', stream_name_neo=None): """ Initialize reader for one or multiple TDT data blocks. @@ -49,6 +48,9 @@ def __init__(self, dirname='', sortname=''): if sortname=='PLX', there should be a ./sort/PLX/*.SortResult file in the tdt block, which stores the sortcode for every spike Default: '', uses the original online sort. + stream_name (str): + If there are several streams, specify the stream name you want to load. + Default: None, load all streams. """ @@ -65,6 +67,7 @@ def __init__(self, dirname='', sortname=''): raise ValueError(f'No data folder or file found for {dirname}') self.sortname = sortname + self.stream_name_neo = stream_name_neo def _source_name(self): return self.dirname @@ -110,7 +113,7 @@ def _parse_header(self): for seg_index, segment_name in enumerate(segment_names): tev_filename = self._get_filestem(segment_name).with_suffix('.tev') if tev_filename.exists(): - tev_data = np.load(tev_filename, mode='r', offset=0, dtype='uint8') + tev_data = np.memmap(tev_filename, mode='r', offset=0, dtype='uint8') else: tev_data = None self._tev_datas.append(tev_data) @@ -179,12 +182,16 @@ def _parse_header(self): self._sigs_t_start = {seg_index: {} for seg_index in range(nb_segment)} # key = seg_index then group_id - keep = info_channel_groups['TankEvType'] == EVTYPE_STREAM + if self.stream_name_neo is not None: + keep = info_channel_groups['StoreName'].astype(str) == self.stream_name_neo + else: + keep = info_channel_groups['TankEvType'] == EVTYPE_STREAM + missing_sev_channels = [] for stream_index, info in enumerate(info_channel_groups[keep]): self._sig_sample_per_chunk[stream_index] = info['NumPoints'] - stream_name = str(info['StoreName']) + stream_name = info['StoreName'].astype(str) stream_id = f'{stream_index}' signal_streams.append((stream_name, stream_id)) @@ -199,7 +206,7 @@ def _parse_header(self): # get data index tsq = self._tsq[seg_index] mask = ((tsq['evtype'] == EVTYPE_STREAM) | \ - (tsq['evtype'] == EVTYPE_STREAM_VARIANT)) & \ + (tsq['evtype'] == int(33041))) & \ (tsq['evname'] == info['StoreName']) & \ (tsq['channel'] == chan_id) data_index = tsq[mask].copy() @@ -250,7 +257,7 @@ def _parse_header(self): sev_filename = (path / sev_stem).with_suffix('.sev') else: # for single block datasets the exact name of sev files in not known - sev_regex = f"*_[cC]h{chan_id}.sev" + sev_regex = f"*_Ch{chan_id}.sev" sev_filename = list(self.dirname.parent.glob(str(sev_regex))) # in case multiple sev files are found, try to find the one for current stream if len(sev_filename) > 1: @@ -266,7 +273,7 @@ def _parse_header(self): sev_filename = sev_filename[0] if (sev_filename is not None) and sev_filename.exists(): - data = np.load(sev_filename, mode='r', offset=0, dtype='uint8') + data = np.memmap(sev_filename, mode='r', offset=0, dtype='uint8') else: data = self._tev_datas[seg_index] assert data is not None, 'no TEV nor SEV' @@ -570,7 +577,6 @@ def read_tbk(tbk_filename): EVTYPE_STROFF = int('00000102', 16) # 258 EVTYPE_SCALAR = int('00000201', 16) # 513 EVTYPE_STREAM = int('00008101', 16) # 33025 -EVTYPE_STREAM_VARIANT = int('00008111', 16) # 33041 EVTYPE_SNIP = int('00008201', 16) # 33281 EVTYPE_MARK = int('00008801', 16) # 34817 EVTYPE_HASDATA = int('00008000', 16) # 32768 @@ -603,4 +609,4 @@ def is_tdtblock(blockpath): if file_ext >= tdt_ext: # if containing all the necessary files return True else: - return False + return False \ No newline at end of file From b99eec456e1f572b04f7aca2e5b8c5d672fc8621 Mon Sep 17 00:00:00 2001 From: JulesLebert Date: Thu, 20 Apr 2023 08:14:27 +0100 Subject: [PATCH 3/5] replace np.memmap with np.fromfile to run on clusters with high RAM and limited maximum allowed number of files open --- neo/rawio/tdtrawio.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/neo/rawio/tdtrawio.py b/neo/rawio/tdtrawio.py index ae945d1e5..fff9595f9 100644 --- a/neo/rawio/tdtrawio.py +++ b/neo/rawio/tdtrawio.py @@ -113,7 +113,8 @@ def _parse_header(self): for seg_index, segment_name in enumerate(segment_names): tev_filename = self._get_filestem(segment_name).with_suffix('.tev') if tev_filename.exists(): - tev_data = np.memmap(tev_filename, mode='r', offset=0, dtype='uint8') + # tev_data = np.memmap(tev_filename, mode='r', offset=0, dtype='uint8') + tev_data = np.fromfile(tev_filename, offset=0, dtype='uint8') else: tev_data = None self._tev_datas.append(tev_data) @@ -273,7 +274,8 @@ def _parse_header(self): sev_filename = sev_filename[0] if (sev_filename is not None) and sev_filename.exists(): - data = np.memmap(sev_filename, mode='r', offset=0, dtype='uint8') + # data = np.memmap(sev_filename, mode='r', offset=0, dtype='uint8') + data = np.fromfile(sev_filename, offset=0, dtype='uint8') else: data = self._tev_datas[seg_index] assert data is not None, 'no TEV nor SEV' From 733f0615bcc8f5c14902db18820b51a2796b1d04 Mon Sep 17 00:00:00 2001 From: Jules Lebert <55235543+JulesLebert@users.noreply.github.com> Date: Tue, 20 Jun 2023 13:06:00 +0100 Subject: [PATCH 4/5] load tdtrawio data with memmap --- neo/rawio/tdtrawio.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/neo/rawio/tdtrawio.py b/neo/rawio/tdtrawio.py index fff9595f9..3ee554318 100644 --- a/neo/rawio/tdtrawio.py +++ b/neo/rawio/tdtrawio.py @@ -113,8 +113,8 @@ def _parse_header(self): for seg_index, segment_name in enumerate(segment_names): tev_filename = self._get_filestem(segment_name).with_suffix('.tev') if tev_filename.exists(): - # tev_data = np.memmap(tev_filename, mode='r', offset=0, dtype='uint8') - tev_data = np.fromfile(tev_filename, offset=0, dtype='uint8') + tev_data = np.memmap(tev_filename, mode='r', offset=0, dtype='uint8') + # tev_data = np.fromfile(tev_filename, offset=0, dtype='uint8') else: tev_data = None self._tev_datas.append(tev_data) @@ -274,8 +274,8 @@ def _parse_header(self): sev_filename = sev_filename[0] if (sev_filename is not None) and sev_filename.exists(): - # data = np.memmap(sev_filename, mode='r', offset=0, dtype='uint8') - data = np.fromfile(sev_filename, offset=0, dtype='uint8') + data = np.memmap(sev_filename, mode='r', offset=0, dtype='uint8') + # data = np.fromfile(sev_filename, offset=0, dtype='uint8') else: data = self._tev_datas[seg_index] assert data is not None, 'no TEV nor SEV' @@ -611,4 +611,4 @@ def is_tdtblock(blockpath): if file_ext >= tdt_ext: # if containing all the necessary files return True else: - return False \ No newline at end of file + return False From 3cd7b6cd4ef569abda16f0752d7dac961bb6430c Mon Sep 17 00:00:00 2001 From: Jules Lebert <55235543+JulesLebert@users.noreply.github.com> Date: Wed, 21 Jun 2023 13:32:54 +0100 Subject: [PATCH 5/5] load data in RAM if memmap doesn't work (when a file is empty/corrupted) --- neo/rawio/tdtrawio.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/neo/rawio/tdtrawio.py b/neo/rawio/tdtrawio.py index 3ee554318..4de4fadf7 100644 --- a/neo/rawio/tdtrawio.py +++ b/neo/rawio/tdtrawio.py @@ -113,8 +113,10 @@ def _parse_header(self): for seg_index, segment_name in enumerate(segment_names): tev_filename = self._get_filestem(segment_name).with_suffix('.tev') if tev_filename.exists(): - tev_data = np.memmap(tev_filename, mode='r', offset=0, dtype='uint8') - # tev_data = np.fromfile(tev_filename, offset=0, dtype='uint8') + try: + tev_data = np.memmap(tev_filename, mode='r', offset=0, dtype='uint8') + except: + tev_data = np.fromfile(tev_filename, offset=0, dtype='uint8') else: tev_data = None self._tev_datas.append(tev_data) @@ -274,8 +276,10 @@ def _parse_header(self): sev_filename = sev_filename[0] if (sev_filename is not None) and sev_filename.exists(): - data = np.memmap(sev_filename, mode='r', offset=0, dtype='uint8') - # data = np.fromfile(sev_filename, offset=0, dtype='uint8') + try: + data = np.memmap(sev_filename, mode='r', offset=0, dtype='uint8') + except: + data = np.fromfile(sev_filename, offset=0, dtype='uint8') else: data = self._tev_datas[seg_index] assert data is not None, 'no TEV nor SEV'