Merge pull request #36 from biomarkersParkinson/refactor

Refactor code

.gitignore

@@ -143,3 +143,6 @@ dmypy.json
 
 #VS Code
 .vscode/
+
+tests/data/tmp_*
+docs/data/tmp_*

docs/basic_reading_and_writing.ipynb

@@ -0,0 +1,237 @@
+{
+ "cells": [
+  {
+   "attachments": {},
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "# Basic reading and writing\n",
+    "\n",
+    "These are some examples on how to read and write TSDF data into and from a numpy array, using the `tsdf` library."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [],
+   "source": [
+    "import tsdf"
+   ]
+  },
+  {
+   "attachments": {},
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "### Load some data"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Data type:\t int16\n",
+      "Data shape:\t (10, 3)\n"
+     ]
+    }
+   ],
+   "source": [
+    "# The name of the data\n",
+    "metadata_path = \"data/example_meta.json\"\n",
+    "binary_filename = \"example_binary.bin\"\n",
+    "\n",
+    "# Multiple metadata files (one for each binary) are loaded into a dictionary\n",
+    "# mapping the binary file name to the metadata object\n",
+    "metadata_dict = tsdf.load_metadata_from_path(metadata_path)\n",
+    "\n",
+    "# Retrieve the metadata object we want, using the name of the binary as key\n",
+    "metadata = metadata_dict[binary_filename]\n",
+    "\n",
+    "# Load the data\n",
+    "data = tsdf.load_binary_from_metadata(metadata)\n",
+    "\n",
+    "# Print some info\n",
+    "print(f\"Data type:\\t {data.dtype}\")\n",
+    "print(f\"Data shape:\\t {data.shape}\")"
+   ]
+  },
+  {
+   "attachments": {},
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "### Perform basic data processing"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Processed data type:\t float32\n",
+      "Data shape:\t\t (10, 3)\n"
+     ]
+    }
+   ],
+   "source": [
+    "# Perform an operation, resulting in a different data type\n",
+    "processed_data_1 = (data / 10).astype('float32')\n",
+    "\n",
+    "# Print some info\n",
+    "print(f\"Processed data type:\\t {processed_data_1.dtype}\")\n",
+    "print(f\"Data shape:\\t\\t {processed_data_1.shape}\")"
+   ]
+  },
+  {
+   "attachments": {},
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "### Write the processed data \n",
+    "Write the processed data in binary format. The call returns the corresponding metadata object."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "File written to data/tmp_example_processed.bin\n"
+     ]
+    }
+   ],
+   "source": [
+    "# The new name of the file\n",
+    "output_bin_filename = \"tmp_example_processed.bin\"\n",
+    "\n",
+    "# Write the data to a new binary file\n",
+    "processed_metadata_1 = tsdf.write_binary_file(\n",
+    "        \"data\",\n",
+    "        output_bin_filename,\n",
+    "        processed_data_1,\n",
+    "        metadata.get_plain_tsdf_dict_copy(),\n",
+    "    )\n",
+    "\n",
+    "print(f\"File written to data/{output_bin_filename}\")"
+   ]
+  },
+  {
+   "attachments": {},
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "### Write the TSDF metadata file"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 5,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "File written to data/tmp_example_processed_meta.json\n"
+     ]
+    }
+   ],
+   "source": [
+    "# Write new metadata file\n",
+    "output_meta_filename = \"tmp_example_processed_meta.json\"\n",
+    "tsdf.write_metadata([processed_metadata_1], output_meta_filename)\n",
+    "print(f\"File written to data/{output_meta_filename}\")"
+   ]
+  },
+  {
+   "attachments": {},
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "### Write a metadata file that combines multiple binary files"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 6,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "File written to data/tmp_example_processed_2.bin\n",
+      "File written to data/tmp_example_processed_2_meta.json\n"
+     ]
+    }
+   ],
+   "source": [
+    "# Preprocess the original data to generate another data source\n",
+    "processed_data_2 = (data * 1000).astype(\"int32\")\n",
+    "\n",
+    "# Adjust the metadata slightly\n",
+    "updated_metadata = metadata.get_plain_tsdf_dict_copy()\n",
+    "updated_metadata.pop(\"scale_factors\")  # remove the 'scale_factors'\n",
+    "\n",
+    "# Save the new binary file\n",
+    "output_bin_filename_2 = \"tmp_example_processed_2.bin\"\n",
+    "processed_metadata_2 = tsdf.write_binary_file(\n",
+    "    \"data\",\n",
+    "    output_bin_filename_2,\n",
+    "    processed_data_2,\n",
+    "    updated_metadata,\n",
+    ")\n",
+    "print(f\"File written to data/{output_bin_filename_2}\")\n",
+    "\n",
+    "# Write a metadata file that combines the two binary files\n",
+    "output_meta_filename_2 = \"tmp_example_processed_2_meta.json\"\n",
+    "tsdf.write_metadata([processed_metadata_1, processed_metadata_2],\n",
+    "    output_meta_filename_2)\n",
+    "print(f\"File written to data/{output_meta_filename_2}\")"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "tsdf-zVA6tG---py3.9",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.9.13"
+  },
+  "orig_nbformat": 4,
+  "vscode": {
+   "interpreter": {
+    "hash": "d1e978847a77d4ff49203fd09f0f7925f58560bf1007438482d75cb657018d9b"
+   }
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}

docs/converting_legacy_data.ipynb

@@ -0,0 +1,70 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Transform legacy (TSDB) format to the current TSDF v0.1\n",
+    "Transform one file (or all files within the given directory) from TSDB to TSDF format."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import os\n",
+    "from tsdf.legacy_tsdf_utils import (\n",
+    "    generate_tsdf_metadata_from_tsdb,\n",
+    "    convert_file_tsdb_to_tsdf,\n",
+    "    convert_files_tsdb_to_tsdf,\n",
+    ")\n",
+    "\n",
+    "data_dir = 'data'"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Path to the metadata file\n",
+    "path_to_file = os.path.join(data_dir, \"ppp_format_meta_legacy.json\")\n",
+    "path_to_new_file = os.path.join(data_dir, \"tmp_ppp_format_meta.json\")\n",
+    "\n",
+    "# Generate a TSDF metadata file from TSDB\n",
+    "generate_tsdf_metadata_from_tsdb(path_to_file, path_to_new_file)\n",
+    "\n",
+    "# Convert a TSDB metadata file to TSDB format\n",
+    "# convert_metadata_tsdb_to_tsdf(path_to_file)\n",
+    "\n",
+    "# Convert all metadata files in the directory from TSDB to TSDF format\n",
+    "# convert_metadatas_tsdb_to_tsdf(path_to_dir)"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "tsdf-zVA6tG---py3.9",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.9.13"
+  },
+  "orig_nbformat": 4
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}

docs/data/example_meta.json

@@ -0,0 +1,30 @@
+{
+    "subject_id": "dummy",
+    "study_id": "dummy",
+    "device_id": "dummy",
+    "endianness": "little",
+    "metadata_version": "0.1",
+    "start_datetime_unix_ms": 1571135957025,
+    "start_iso8601": "2019-10-15T10:39:17.025000+00:00",
+    "end_datetime_unix_ms": 1571168851826,
+    "end_iso8601": "2019-10-15T19:47:31.826000+00:00",
+    "file_name": "example_binary.bin",
+    "channels": [
+        "x",
+        "y",
+        "z"
+    ],
+    "units": [
+        "m/s/s",
+        "m/s/s",
+        "m/s/s"
+    ],
+    "scale_factors": [
+        0.00469378,
+        0.00469378,
+        0.00469378
+    ],
+    "data_type": "int",
+    "bits": 16,
+    "rows": 10
+}