diff --git a/bw_processing/constants.py b/bw_processing/constants.py index 09f102f..71ff749 100644 --- a/bw_processing/constants.py +++ b/bw_processing/constants.py @@ -4,6 +4,7 @@ import numpy as np MAX_SIGNED_32BIT_INT = 2147483647 +MAX_SIGNED_64BIT_INT = 9223372036854775807 # We could try to save space by not storing the columns # `row_index` and `col_index`, and add them after loading from @@ -20,7 +21,7 @@ ("maximum", np.float32), ("negative", bool), ] -INDICES_DTYPE = [("row", np.int32), ("col", np.int32)] +INDICES_DTYPE = [("row", np.int64), ("col", np.int64)] NAME_RE = re.compile(r"^[\w\-\.]*$") diff --git a/bw_processing/datapackage.py b/bw_processing/datapackage.py index 471a419..6daa686 100644 --- a/bw_processing/datapackage.py +++ b/bw_processing/datapackage.py @@ -14,6 +14,8 @@ from .constants import ( DEFAULT_LICENSES, INDICES_DTYPE, + MAX_SIGNED_32BIT_INT, + MAX_SIGNED_64BIT_INT, NUMPY_SERIALIZE_FORMAT_EXTENSION, NUMPY_SERIALIZE_FORMAT_NAME, PARQUET_SERIALIZE_FORMAT_EXTENSION, @@ -257,6 +259,12 @@ def rehydrate_interface( self.data[index] = resource + def get_max_index_value(self) -> int: + """Get maximum index value (max signed 32 or 64 bit integer) for this datapackage""" + if self.metadata.get("64_bit_indices"): + return MAX_SIGNED_64BIT_INT + return MAX_SIGNED_32BIT_INT + class FilteredDatapackage(DatapackageBase): """A subset of a datapackage. Used in matrix construction or other data manipulation operations. @@ -360,6 +368,7 @@ def _create( "combinatorial": combinatorial, "sequential": sequential, "seed": seed, + "64_bit_indices": True, "sum_intra_duplicates": sum_intra_duplicates, "sum_inter_duplicates": sum_inter_duplicates, "matrix_serialize_format_type": matrix_serialize_format_type.value, diff --git a/tests/fixtures/indexing/array.indices.npy b/tests/fixtures/indexing/array.indices.npy index f8634de..3826c0f 100644 Binary files a/tests/fixtures/indexing/array.indices.npy and b/tests/fixtures/indexing/array.indices.npy differ diff --git a/tests/fixtures/indexing/datapackage.json b/tests/fixtures/indexing/datapackage.json index ecf1fe5..bbb230f 100644 --- a/tests/fixtures/indexing/datapackage.json +++ b/tests/fixtures/indexing/datapackage.json @@ -50,10 +50,10 @@ "profile": "data-resource", "format": "npy", "mediatype": "application/octet-stream", - "name": "array.data", + "name": "array.indices", "matrix": "sa_matrix", - "kind": "data", - "path": "array.data.npy", + "kind": "indices", + "path": "array.indices.npy", "group": "array", "category": "array", "nrows": 3 @@ -62,10 +62,10 @@ "profile": "data-resource", "format": "npy", "mediatype": "application/octet-stream", - "name": "array.indices", + "name": "array.data", "matrix": "sa_matrix", - "kind": "indices", - "path": "array.indices.npy", + "kind": "data", + "path": "array.data.npy", "group": "array", "category": "array", "nrows": 3 @@ -167,10 +167,11 @@ ] } ], - "created": "2021-05-18T13:12:19.590666Z", + "created": "2024-08-18T20:58:57.736902Z", "combinatorial": false, "sequential": false, "seed": null, "sum_intra_duplicates": true, - "sum_inter_duplicates": false -} + "sum_inter_duplicates": false, + "matrix_serialize_format_type": "numpy" +} \ No newline at end of file diff --git a/tests/fixtures/indexing/vector.data.npy b/tests/fixtures/indexing/vector.data.npy index abb6ffd..8c300fb 100644 Binary files a/tests/fixtures/indexing/vector.data.npy and b/tests/fixtures/indexing/vector.data.npy differ diff --git a/tests/fixtures/indexing/vector.indices.npy b/tests/fixtures/indexing/vector.indices.npy index f8634de..3826c0f 100644 Binary files a/tests/fixtures/indexing/vector.indices.npy and b/tests/fixtures/indexing/vector.indices.npy differ diff --git a/tests/fixtures/merging/merging_first.zip b/tests/fixtures/merging/merging_first.zip index 7182312..5d2425c 100644 Binary files a/tests/fixtures/merging/merging_first.zip and b/tests/fixtures/merging/merging_first.zip differ diff --git a/tests/fixtures/merging/merging_same_1.zip b/tests/fixtures/merging/merging_same_1.zip index 6e7d713..61834c8 100644 Binary files a/tests/fixtures/merging/merging_same_1.zip and b/tests/fixtures/merging/merging_same_1.zip differ diff --git a/tests/fixtures/merging/merging_same_2.zip b/tests/fixtures/merging/merging_same_2.zip index 0355738..feae666 100644 Binary files a/tests/fixtures/merging/merging_same_2.zip and b/tests/fixtures/merging/merging_same_2.zip differ diff --git a/tests/fixtures/merging/merging_second.zip b/tests/fixtures/merging/merging_second.zip index d239b5e..45041c4 100644 Binary files a/tests/fixtures/merging/merging_second.zip and b/tests/fixtures/merging/merging_second.zip differ diff --git a/tests/fixtures/test-fixture.zip b/tests/fixtures/test-fixture.zip index 7d4c1aa..4140734 100644 Binary files a/tests/fixtures/test-fixture.zip and b/tests/fixtures/test-fixture.zip differ diff --git a/tests/fixtures/tfd/datapackage.json b/tests/fixtures/tfd/datapackage.json index 9c04974..6cb5f94 100644 --- a/tests/fixtures/tfd/datapackage.json +++ b/tests/fixtures/tfd/datapackage.json @@ -203,7 +203,7 @@ "valid_for": "sa-data-array" } ], - "created": "2024-05-22T12:20:16.587994+00:00Z", + "created": "2024-08-18T20:58:29.941013Z", "combinatorial": false, "sequential": false, "seed": null, diff --git a/tests/fixtures/tfd/sa-array-interface.indices.npy b/tests/fixtures/tfd/sa-array-interface.indices.npy index 08fa6ec..759389f 100644 Binary files a/tests/fixtures/tfd/sa-array-interface.indices.npy and b/tests/fixtures/tfd/sa-array-interface.indices.npy differ diff --git a/tests/fixtures/tfd/sa-data-array.indices.npy b/tests/fixtures/tfd/sa-data-array.indices.npy index 08fa6ec..759389f 100644 Binary files a/tests/fixtures/tfd/sa-data-array.indices.npy and b/tests/fixtures/tfd/sa-data-array.indices.npy differ diff --git a/tests/fixtures/tfd/sa-data-vector-from-dict.indices.npy b/tests/fixtures/tfd/sa-data-vector-from-dict.indices.npy index ad5e19f..b806140 100644 Binary files a/tests/fixtures/tfd/sa-data-vector-from-dict.indices.npy and b/tests/fixtures/tfd/sa-data-vector-from-dict.indices.npy differ diff --git a/tests/fixtures/tfd/sa-data-vector.indices.npy b/tests/fixtures/tfd/sa-data-vector.indices.npy index 08fa6ec..759389f 100644 Binary files a/tests/fixtures/tfd/sa-data-vector.indices.npy and b/tests/fixtures/tfd/sa-data-vector.indices.npy differ diff --git a/tests/fixtures/tfd/sa-vector-interface.indices.npy b/tests/fixtures/tfd/sa-vector-interface.indices.npy index 08fa6ec..759389f 100644 Binary files a/tests/fixtures/tfd/sa-vector-interface.indices.npy and b/tests/fixtures/tfd/sa-vector-interface.indices.npy differ