Thanks to visit codestin.com
Credit goes to github.com

Skip to content

Use BigInt for shape, maxshape and total_size in metadata #112

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
bmaranville wants to merge 5 commits into from
Closed

Use BigInt for shape, maxshape and total_size in metadata #112

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
e64bf55
return hsize_t (bigint) for shape, maxshape and total_size in metadata
bmaranville Dec 16, 2025
75b9e23
add check_malloc and handle metadata.shape and total_size etc. as bigint
bmaranville Dec 16, 2025
7f933f6
compiled types with new bigint for shape
bmaranville Dec 16, 2025
ecdddbf
update interface for Metadata to include bigint shape, maxshape, tota…
bmaranville Dec 16, 2025
4dd75b7
update tests to use bigint for shape, maxshape, total_size
bmaranville Dec 16, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 4 additions & 3 deletions src/hdf5_hl.d.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,8 @@ declare type TypedArray = Int8Array | Uint8Array | Uint8ClampedArray | Int16Arra
* `[i0, i1]` - select all data in the range `i0` to `i1`
* `[i0, i1, s]` - select every `s` values in the range `i0` to `i1`
**/
declare type Slice = [] | [number | null] | [number | null, number | null] | [number | null, number | null, number | null];
declare type SliceElement = bigint | number | null;
declare type Slice = [] | [SliceElement] | [SliceElement, SliceElement] | [SliceElement, SliceElement, SliceElement];
export declare type GuessableDataTypes = TypedArray | number | number[] | string | string[] | Reference | Reference[] | RegionReference | RegionReference[];
declare enum OBJECT_TYPE {
DATASET = "Dataset",
Expand Down Expand Up @@ -63,7 +64,7 @@ export declare class Attribute {
name: string;
metadata: Metadata;
dtype: Dtype;
shape: number[] | null;
shape: bigint[] | null;
private _value?;
private _json_value?;
constructor(file_id: bigint, path: string, name: string);
Expand Down Expand Up @@ -133,7 +134,7 @@ export declare class Dataset extends HasAttrs {
refresh(): void;
get metadata(): Metadata;
get dtype(): Dtype;
get shape(): number[] | null;
get shape(): bigint[] | null;
get filters(): Filter[];
get value(): OutputData | null;
get json_value(): JSONCompatibleOutputData | null;
Expand Down
87 changes: 57 additions & 30 deletions src/hdf5_hl.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,19 @@ function dirname(path: string) {
return head;
}

function check_malloc(nbytes: bigint | number) {
const max_memory = Module.MAXIMUM_MEMORY;
if (nbytes > max_memory) {
throw new Error(`Requested allocation of ${nbytes} bytes exceeds maximum memory of ${max_memory} bytes`);
}
const safe_nbytes = Number(nbytes);
const ptr = Module._malloc(safe_nbytes);
if (ptr === 0) {
throw new Error(`Memory allocation of ${safe_nbytes} bytes failed`);
}
return ptr;
}

function get_attr(file_id: bigint, obj_name: string, attr_name: string, json_compatible: true): JSONCompatibleOutputData | null;
function get_attr(file_id: bigint, obj_name: string, attr_name: string, json_compatible: false): OutputData | null;
function get_attr(file_id: bigint, obj_name: string, attr_name: string, json_compatible: boolean): OutputData | JSONCompatibleOutputData | null;
Expand All @@ -49,12 +62,12 @@ function get_attr(file_id: bigint, obj_name: string, attr_name: string, json_com
return null;
}

let nbytes = metadata.size * metadata.total_size;
let data_ptr = Module._malloc(nbytes);
const nbytes = BigInt(metadata.size) * metadata.total_size;
let data_ptr = check_malloc(nbytes);
var processed;
try {
Module.get_attribute_data(file_id, obj_name, attr_name, BigInt(data_ptr));
let data = Module.HEAPU8.slice(data_ptr, data_ptr + nbytes);
let data = Module.HEAPU8.slice(data_ptr, data_ptr + Number(nbytes));
processed = process_data(data, metadata, json_compatible);
} finally {
if (metadata.vlen) {
Expand Down Expand Up @@ -146,7 +159,7 @@ function process_data(data: Uint8Array, metadata: Metadata, json_compatible: boo
else if (type === Module.H5T_class_t.H5T_INTEGER.value || type === Module.H5T_class_t.H5T_FLOAT.value) {
const { size, signed} = metadata;
const accessor = getAccessor(type, size, signed);
output_data = new accessor(data.buffer);
output_data = new accessor(data.buffer as ArrayBuffer);
if (json_compatible) {
output_data = [...output_data];
if (accessor === BigInt64Array || accessor === BigUint64Array) {
Expand All @@ -173,12 +186,15 @@ function process_data(data: Uint8Array, metadata: Metadata, json_compatible: boo

else if (type === Module.H5T_class_t.H5T_ARRAY.value) {
const { array_type } = <{array_type: Metadata}>metadata;
shape = (<number[]>shape).concat(<number[]>array_type.shape);
shape = (<bigint[]>shape).concat(<bigint[]>array_type.shape);
array_type.shape = shape;
// always convert ARRAY types to base JS types:
output_data = process_data(data, array_type, true);
if (isIterable(output_data) && typeof output_data !== "string") {
output_data = create_nested_array(output_data as JSONCompatibleOutputData[], array_type.shape);
// because process_data is called after data is already retrievied, we know that
// total_size < Module.MAXIMUM_MEMORY, so it's safe to convert shape to number[]
const output_shape = array_type.shape.map(Number);
output_data = create_nested_array(output_data as JSONCompatibleOutputData[], output_shape);
}
}

Expand All @@ -201,7 +217,9 @@ function process_data(data: Uint8Array, metadata: Metadata, json_compatible: boo
else if (type === Module.H5T_class_t.H5T_REFERENCE.value) {
const { ref_type, size } = metadata; // as { ref_type: 'object' | 'region', size: number };
const cls = (ref_type === 'object') ? Reference : RegionReference;
output_data = Array.from({ length: metadata.total_size }).map((_, i) => {
// because process_data is called after data is already retrievied, we know that
// total_size < Module.MAXIMUM_MEMORY, so it's safe to convert length to number
output_data = Array.from({ length: Number(metadata.total_size) }).map((_, i) => {
const ref_data = data.slice(i*size, (i+1)*size);
return new cls(ref_data);
});
Expand All @@ -226,7 +244,7 @@ function process_data(data: Uint8Array, metadata: Metadata, json_compatible: boo
const data = Module.HEAPU8.slice(data_ptr, data_ptr + data_nbytes);

// Process this vlen array's data according to base datatype
output.push(process_data(data, { ...vlen_type, shape: [length], total_size: length }, json_compatible));
output.push(process_data(data, { ...vlen_type, shape: [BigInt(length)], total_size: BigInt(length) }, json_compatible));
}

output_data = output;
Expand Down Expand Up @@ -461,7 +479,8 @@ const TypedArray_to_dtype = new Map([
* `[i0, i1]` - select all data in the range `i0` to `i1`
* `[i0, i1, s]` - select every `s` values in the range `i0` to `i1`
**/
type Slice = [] | [number|null] | [number|null,number|null] | [number|null, number|null, number|null];
type SliceElement = bigint | number | null;
type Slice = [] | [SliceElement] | [SliceElement, SliceElement] | [SliceElement, SliceElement, SliceElement];

export type GuessableDataTypes = TypedArray | number | number[] | string | string[] | Reference | Reference[] | RegionReference | RegionReference[];

Expand Down Expand Up @@ -542,7 +561,7 @@ export class Attribute {
name: string;
metadata: Metadata;
dtype: Dtype;
shape: number[] | null;
shape: bigint[] | null;
private _value?: OutputData | null;
private _json_value?: JSONCompatibleOutputData | null;

Expand Down Expand Up @@ -573,10 +592,10 @@ export class Attribute {
to_array(): JSONCompatibleOutputData | null {
const { json_value, metadata } = this;
const { shape } = metadata;
if (!isIterable(json_value) || typeof json_value === "string") {
if (!isIterable(json_value) || typeof json_value === "string" || shape === null) {
return json_value;
}
return create_nested_array(json_value, <number[]>shape);
return create_nested_array(json_value, shape.map(Number));
}
}

Expand Down Expand Up @@ -635,7 +654,7 @@ abstract class HasAttrs {
);
}
else {
let data_ptr = Module._malloc((prepared_data as Uint8Array).byteLength);
let data_ptr = check_malloc((prepared_data as Uint8Array).byteLength);
try {
Module.HEAPU8.set(prepared_data as Uint8Array, data_ptr);
Module.create_attribute(
Expand Down Expand Up @@ -830,7 +849,7 @@ export class Group extends HasAttrs {
);
}
else {
let data_ptr = Module._malloc((prepared_data as Uint8Array).byteLength);
let data_ptr = check_malloc((prepared_data as Uint8Array).byteLength);
try {
Module.HEAPU8.set(prepared_data as Uint8Array, data_ptr);
Module.create_dataset(
Expand Down Expand Up @@ -904,14 +923,22 @@ export class File extends Group {
}
}

const calculateHyperslabParams = (shape: number[],ranges: Slice[]) => {
const calculateHyperslabParams = (shape: bigint[], ranges: Slice[]) => {
const strides = shape.map((s, i) => BigInt(ranges?.[i]?.[2] ?? 1));
const count = shape.map((s, i) => {
const N = BigInt((Math.min(s, ranges?.[i]?.[1] ?? s) - Math.max(0, ranges?.[i]?.[0] ?? 0)));
const range_upper = ranges?.[i]?.[1] ?? s;
const range_lower = ranges?.[i]?.[0] ?? 0n;
const high = (range_upper < s) ? BigInt(range_upper) : s;
const low = (range_lower > 0n) ? BigInt(range_lower) : 0n;
const N = high - low;
const st = strides[i];
return N / st + ((N % st) + st - 1n)/st
});
const offset = shape.map((s, i) => BigInt(Math.min(s, Math.max(0, ranges?.[i]?.[0] ?? 0))));
const offset = shape.map((s, i) => {
const range_lower = ranges?.[i]?.[0] ?? 0n;
const low = (range_lower > 0n) ? BigInt(range_lower) : 0n;
return (s < low) ? s : low;
});
return {strides, count, offset}
}

Expand Down Expand Up @@ -971,12 +998,12 @@ export class Dataset extends HasAttrs {

const {strides, count, offset} = calculateHyperslabParams(shape, ranges);
const total_size = count.reduce((previous, current) => current * previous, 1n);
const nbytes = metadata.size * Number(total_size);
const data_ptr = Module._malloc(nbytes);
const nbytes = BigInt(metadata.size) * total_size;
const data_ptr = check_malloc(nbytes);
let processed: OutputData;
try {
Module.get_dataset_data(this.file_id, this.path, count, offset, strides, BigInt(data_ptr));
let data = Module.HEAPU8.slice(data_ptr, data_ptr + nbytes);
let data = Module.HEAPU8.slice(data_ptr, data_ptr + Number(nbytes));
processed = process_data(data, metadata, false);
} finally {
if (metadata.vlen || metadata.type === Module.H5T_class_t.H5T_VLEN.value) {
Expand All @@ -1000,8 +1027,8 @@ export class Dataset extends HasAttrs {
// if auto_refresh is on, getting the metadata has triggered a refresh of the dataset_id;
const {strides, count, offset} = calculateHyperslabParams(shape, ranges);

const { data: prepared_data, shape: guessed_shape } = prepare_data(data, metadata, count);
let data_ptr = Module._malloc((prepared_data as Uint8Array).byteLength);
const { data: prepared_data } = prepare_data(data, metadata, count);
let data_ptr = check_malloc((prepared_data as Uint8Array).byteLength);
Module.HEAPU8.set(prepared_data as Uint8Array, data_ptr);

try {
Expand All @@ -1028,10 +1055,10 @@ export class Dataset extends HasAttrs {
to_array(): JSONCompatibleOutputData | null {
const { json_value, metadata } = this;
const { shape } = metadata;
if (!isIterable(json_value) || typeof json_value === "string") {
if (!isIterable(json_value) || typeof json_value === "string" || shape === null) {
return json_value;
}
let nested = create_nested_array(json_value, <number[]>shape);
let nested = create_nested_array(json_value, shape.map(Number));
return nested;
}

Expand Down Expand Up @@ -1090,12 +1117,12 @@ export class Dataset extends HasAttrs {
}

// if auto_refresh is on, getting the metadata has triggered a refresh of the dataset_id;
let nbytes = metadata.size * metadata.total_size;
let data_ptr = Module._malloc(nbytes);
let nbytes = BigInt(metadata.size) * metadata.total_size;
let data_ptr = check_malloc(nbytes);
let processed: OutputData;
try {
Module.get_dataset_data(this.file_id, this.path, null, null, null, BigInt(data_ptr));
let data = Module.HEAPU8.slice(data_ptr, data_ptr + nbytes);
let data = Module.HEAPU8.slice(data_ptr, data_ptr + Number(nbytes));
processed = process_data(data, metadata, json_compatible);
} finally {
if (metadata.vlen) {
Expand Down Expand Up @@ -1139,12 +1166,12 @@ export class DatasetRegion {
}

// if auto_refresh is on, getting the metadata has triggered a refresh of the dataset_id;
let nbytes = metadata.size * metadata.total_size;
let data_ptr = Module._malloc(nbytes);
let nbytes = BigInt(metadata.size) * metadata.total_size;
let data_ptr = check_malloc(nbytes);
let processed: OutputData;
try {
Module.get_region_data(this.source_dataset.file_id, this.region_reference.ref_data, BigInt(data_ptr));
let data = Module.HEAPU8.slice(data_ptr, data_ptr + nbytes);
let data = Module.HEAPU8.slice(data_ptr, data_ptr + Number(nbytes));
processed = process_data(data, metadata, json_compatible);
} finally {
if (metadata.vlen) {
Expand Down
22 changes: 12 additions & 10 deletions src/hdf5_util.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@
#include "H5PLextern.h"
#include <emscripten/bind.h>
#include <emscripten.h>
#include <emscripten/heap.h>

#define ATTRIBUTE_DATA 0
#define DATASET_DATA 1
Expand Down Expand Up @@ -329,10 +330,10 @@ val get_dtype_metadata(hid_t dtype)
int ndims = H5Tget_array_ndims(dtype);
std::vector<hsize_t> array_dims(ndims);
H5Tget_array_dims2(dtype, &array_dims[0]);
int total_size = 1;
hsize_t total_size = 1;
for (int i=0; i<ndims; i++) {
array_dims_out.set(i, (int)array_dims[i]);
total_size *= (int)array_dims[i];
array_dims_out.set(i, array_dims[i]);
total_size *= array_dims[i];
}
array_type.set("shape", array_dims_out);
array_type.set("total_size", total_size);
Expand Down Expand Up @@ -402,7 +403,7 @@ val get_abstractDS_metadata(hid_t dspace, hid_t dtype, hid_t dcpl)
val attr = get_dtype_metadata(dtype);

int type = H5Sget_simple_extent_type(dspace);
int total_size = H5Sget_simple_extent_npoints(dspace);
hsize_t total_size = H5Sget_simple_extent_npoints(dspace);
attr.set("total_size", total_size);

if (type == H5S_NULL) {
Expand All @@ -421,8 +422,8 @@ val get_abstractDS_metadata(hid_t dspace, hid_t dtype, hid_t dcpl)
val shape = val::array();
val maxshape = val::array();
for (int d = 0; d < ndims; d++) {
shape.set(d, (uint)dims_out.at(d));
maxshape.set(d, (uint)maxdims_out.at(d));
shape.set(d, dims_out.at(d));
maxshape.set(d, maxdims_out.at(d));
}

attr.set("shape", shape);
Expand Down Expand Up @@ -1224,7 +1225,7 @@ val get_region_metadata(hid_t loc_id, const val ref_data_in)
// fill in shape, maxshape, chunks, total_size
val metadata = get_abstractDS_metadata(dspace, dtype, dcpl);
// then override the ones that are specific to a region:
int total_size = H5Sget_select_npoints(dspace);
hsize_t total_size = H5Sget_select_npoints(dspace);
metadata.set("total_size", total_size);

int rank = H5Sget_simple_extent_ndims(dspace);
Expand All @@ -1236,11 +1237,11 @@ val get_region_metadata(hid_t loc_id, const val ref_data_in)
std::vector<hsize_t> count(rank);
std::vector<hsize_t> block(rank);
htri_t success = H5Sget_regular_hyperslab(dspace, nullptr, nullptr, count.data(), block.data());
shape = val::array();
shape = val::array(); // elements of type hsize_t
for (int d = 0; d < rank; d++)
{
int blocksize = (block.at(d) == NULL) ? 1 : block.at(d);
shape.set(d, (uint)(count.at(d) * blocksize));
hsize_t blocksize = (block.at(d) == NULL) ? 1 : block.at(d);
shape.set(d, (count.at(d) * blocksize));
}
}
metadata.set("shape", shape);
Expand Down Expand Up @@ -1442,6 +1443,7 @@ EMSCRIPTEN_BINDINGS(hdf5)
constant("H5Z_FILTER_SCALEOFFSET", H5Z_FILTER_SCALEOFFSET);
constant("H5Z_FILTER_RESERVED", H5Z_FILTER_RESERVED);
constant("H5Z_FILTER_MAX", H5Z_FILTER_MAX);
constant("MAXIMUM_MEMORY", emscripten_get_heap_max());

register_vector<std::string>("vector<string>");
}
Expand Down
7 changes: 4 additions & 3 deletions src/hdf5_util_helpers.d.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,13 +25,13 @@ export interface Metadata {
enum_type?: EnumTypeMetadata,
vlen_type?: Metadata,
littleEndian: boolean,
maxshape: number[] | null,
maxshape: bigint[] | null,
ref_type?: 'object' | 'region',
shape: number[] | null,
shape: bigint[] | null,
signed: boolean,
size: number,
strpad?: number,
total_size: number,
total_size: bigint,
type: number,
virtual_sources?: VirtualSource[],
vlen: boolean,
Expand Down Expand Up @@ -93,6 +93,7 @@ export interface H5Module extends EmscriptenModule {
H5Z_FILTER_SCALEOFFSET: 6;
H5Z_FILTER_RESERVED: 256;
H5Z_FILTER_MAX: 65535;
MAXIMUM_MEMORY: number;
create_group(file_id: bigint, name: string, track_order?: boolean): number;
create_vlen_str_dataset(file_id: bigint, dset_name: string, prepared_data: any, shape: bigint[], maxshape: (bigint | null)[], chunks: bigint[] | null, type: number, size: number, signed: boolean, vlen: boolean, track_order?: boolean): number;
get_dataset_data(file_id: bigint, path: string, count: bigint[] | null, offset: bigint[] | null, strides: bigint[] | null, rdata_ptr: bigint): number;
Expand Down
6 changes: 3 additions & 3 deletions test/bool_test.mjs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,11 +28,11 @@ async function bool_test() {
}
},
littleEndian: true,
maxshape: [2, 2],
shape: [2, 2],
maxshape: [2n, 2n],
shape: [2n, 2n],
signed: true,
size: 1,
total_size: 4,
total_size: 4n,
type: 8,
vlen: false,
});
Expand Down
Loading
Loading