Thanks to visit codestin.com
Credit goes to github.com

Skip to content

Increase Robustness of NIMS Reader #303

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 1 commit into from
Sep 26, 2025
Merged

Increase Robustness of NIMS Reader #303

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion mth5/clients/zen.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -137,7 +137,7 @@ def make_mth5_from_zen(self, survey_id=None, combine=True, **kwargs):
run_list.append(run_group.to_runts())
if combine:
# Combine runs and down sample to 1 second.
combined_run = run_list[0].merge(run_list[1:], new_sample_rate=1)
combined_run = run_list[0].merge(run_list[1:], new_sample_rate=8)
combined_run.run_metadata.id = "sr1_0001"
combined_run_group = station_group.add_run("sr1_0001")
combined_run_group.from_runts(combined_run)
Expand Down
5 changes: 4 additions & 1 deletion mth5/io/nims/header.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -187,7 +187,10 @@ def parse_header_dict(self, header_dict=None):
self.header_gps_longitude = self._get_longitude(
gps_list[4], gps_list[5]
)
self.header_gps_elevation = float(gps_list[6])
if gps_list[6] == "M":
self.header_gps_elevation = 0.0
else:
self.header_gps_elevation = float(gps_list[6])
elif "run" in key:
self.run_id = value.replace('"', "")
else:
Expand Down
30 changes: 14 additions & 16 deletions mth5/io/nims/nims.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -509,6 +509,8 @@ def station_metadata(self):
"""Station metadata from nims file"""
if self.ts_data is not None:
station_metadata = Station()
if self.run_id is None:
self.run_id = f"001"
station_metadata.from_dict(
{
"Station": {
Expand Down Expand Up @@ -832,13 +834,18 @@ def remove_duplicates(self, info_array, data_array):
info_array[d["sequence_index"] + 1],
):
duplicate_list.append(d)

if (len(duplicate_list) == 0):
self.logger.info(f"Duplicate block count is zero")
return info_array, data_array, None
self.logger.debug(f"Deleting {len(duplicate_list)} duplicate blocks")
### get the index of the blocks to be removed, namely the 1st duplicate
### block
remove_sequence_index = [d["sequence_index"] for d in duplicate_list]
remove_data_index = np.array(
[np.arange(d["ts_index_0"], d["ts_index_1"], 1) for d in duplicate_list]
).flatten()
remove_data_index = np.concatenate([
np.arange(d["ts_index_0"], d["ts_index_1"]) for d in duplicate_list
]).astype(int)

### remove the data
return_info_array = np.delete(info_array, remove_sequence_index)
return_data_array = np.delete(data_array, remove_data_index)
Expand Down Expand Up @@ -907,7 +914,7 @@ def read_nims(self, fn=None):
data = np.frombuffer(self._raw_string, dtype=np.uint8)

### need to make sure that the data starts with a full block
find_first = self.find_sequence(data[0 : self.block_size * 5])[0]
find_first = self.find_sequence(data[0 : self.block_size * 10])[0]
data = data[find_first:]

### get GPS stamps from the binary string first
Expand Down Expand Up @@ -944,7 +951,7 @@ def read_nims(self, fn=None):
for key, index in self._block_dict.items():
if "temp" in key:
# compute temperature
t_value = data[:, index[0]] * 256 + data[:, index[1]]
t_value = data[:, index[0]].astype(np.int32) * 256 + data[:, index[1]].astype(np.int32)

# something to do with the bits where you have to subtract
t_value[np.where(t_value > 32768)] -= 65536
Expand Down Expand Up @@ -972,9 +979,9 @@ def read_nims(self, fn=None):
channel_arr = np.zeros((data.shape[0], 8), dtype=float)
for kk in range(self.sample_rate):
index = self.indices[kk, cc]
value = (data[:, index] * 256 + data[:, index + 1]) * np.array(
value = (data[:, index].astype(np.int32) * 256 + data[:, index + 1].astype(np.int32)) * np.array(
[256]
) + data[:, index + 2]
) + data[:, index + 2].astype(np.int32)
value[np.where(value > self._int_max)] -= self._int_factor
channel_arr[:, kk] = value
data_array[comp][:] = channel_arr.flatten()
Expand Down Expand Up @@ -1100,15 +1107,6 @@ def align_data(self, data_array, stamps):
### check timing first to make sure there is no drift
timing_valid, self.gaps, time_difference = self.check_timing(stamps)

### need to trim off the excess number of points that are present because of
### data gaps. This will be the time difference times the sample rate
if time_difference > 0:
remove_points = int(time_difference * self.sample_rate)
data_array = data_array[0:-remove_points]
self.logger.info(
f"Trimmed {remove_points} points off the end of the time "
"series because of timing gaps"
)
### first GPS stamp within the data is at a given index that is
### assumed to be the number of seconds from the start of the run.
### therefore make the start time the first GPS stamp time minus
Expand Down
6 changes: 3 additions & 3 deletions mth5/io/zen/zen.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -307,15 +307,15 @@ def sample_rate(self, sampling_rate):
def start(self):
if self.gps_stamps is not None:
return self.get_UTC_date_time(
self.header.gpsweek, self.gps_stamps["time"][0]
self.header.gpsweek, float(self.gps_stamps["time"][0])
)
return self.zen_schedule

@property
def end(self):
if self.gps_stamps is not None:
return self.get_UTC_date_time(
self.header.gpsweek, self.gps_stamps["time"][-1]
self.header.gpsweek, float(self.gps_stamps["time"][-1])
)
return self.start + (self.n_samples / self.sample_rate)

Expand Down Expand Up @@ -972,7 +972,7 @@ def check_start_time(self):

# make sure the time is in gps time
zen_start_utc = self.get_UTC_date_time(
self.header.gpsweek, self.gps_stamps["time"][0]
self.header.gpsweek, float(self.gps_stamps["time"][0])
)

# estimate the time difference between the two
Expand Down
Loading