From b64f17ffc9186af858fa18136008bd7b63e35525 Mon Sep 17 00:00:00 2001 From: bruno-f-cruz <7049351+bruno-f-cruz@users.noreply.github.com> Date: Thu, 9 May 2024 08:35:57 -0700 Subject: [PATCH 1/4] Add new constructor methods --- harp/reader.py | 112 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 112 insertions(+) diff --git a/harp/reader.py b/harp/reader.py index 6b3645c..a222c3a 100644 --- a/harp/reader.py +++ b/harp/reader.py @@ -13,6 +13,7 @@ from harp.model import BitMask, GroupMask, Model, PayloadMember, Register from harp.io import MessageType, read from harp.schema import read_schema +import requests @dataclass @@ -75,6 +76,117 @@ def __dir__(self) -> Iterable[str]: def __getattr__(self, __name: str) -> RegisterReader: return self.registers[__name] + @staticmethod + def from_file( + filepath: PathLike, + base_path: Optional[PathLike] = None, + include_common_registers: bool = True, + epoch: Optional[datetime] = None, + keep_type: bool = False) -> "DeviceReader": + + device = read_schema(filepath, include_common_registers) + if base_path is None: + path = Path(filepath).absolute().resolve() + base_path = path.parent / device.device + else: + base_path = Path(base_path).absolute().resolve() / device.device + + reg_readers = { + name: _create_register_parser( + device, name, _ReaderParams(base_path, epoch, keep_type) + ) + for name in device.registers.keys() + } + return DeviceReader(device, reg_readers) + + @staticmethod + def from_url( + url: str, + base_path: Optional[PathLike] = None, + include_common_registers: bool = True, + epoch: Optional[datetime] = None, + keep_type: bool = False, + timeout: int = 5) -> "DeviceReader": + + response = requests.get(url, timeout=timeout) + text = response.text + + device = read_schema(text, include_common_registers) + if base_path is None: + base_path = Path(device.device).absolute().resolve() + else: + base_path = Path(base_path).absolute().resolve() + + reg_readers = { + name: _create_register_parser( + device, name, _ReaderParams(base_path, epoch, keep_type) + ) + for name in device.registers.keys() + } + return DeviceReader(device, reg_readers) + + @staticmethod + def from_str( + schema: str, + base_path: Optional[PathLike] = None, + include_common_registers: bool = True, + epoch: Optional[datetime] = None, + keep_type: bool = False) -> "DeviceReader": + + device = read_schema(schema, include_common_registers) + if base_path is None: + base_path = Path(device.device).absolute().resolve() + else: + base_path = Path(base_path).absolute().resolve() + + reg_readers = { + name: _create_register_parser( + device, name, _ReaderParams(base_path, epoch, keep_type) + ) + for name in device.registers.keys() + } + return DeviceReader(device, reg_readers) + + @staticmethod + def from_model( + model: Model, + base_path: Optional[PathLike] = None, + epoch: Optional[datetime] = None, + keep_type: bool = False) -> "DeviceReader": + + if base_path is None: + base_path = Path(model.device).absolute().resolve() + else: + base_path = Path(base_path).absolute().resolve() + + reg_readers = { + name: _create_register_parser( + model, name, _ReaderParams(base_path, epoch, keep_type) + ) + for name in model.registers.keys() + } + return DeviceReader(model, reg_readers) + + @staticmethod + def from_dataset( + dataset: PathLike, + include_common_registers: bool = True, + epoch: Optional[datetime] = None, + keep_type: bool = False) -> "DeviceReader": + + path = Path(dataset).absolute().resolve() + is_dir = os.path.isdir(path) + if is_dir: + filepath = path / "device.yml" + return DeviceReader.from_file( + filepath=filepath, + base_path=path, + include_common_registers=include_common_registers, + epoch=epoch, + keep_type=keep_type) + else: + raise ValueError("The dataset must be a directory containing a device.yml file.") + def _compose_parser( f: Callable[[DataFrame], DataFrame], From ebd1956cd85889f2fe0e871fe1f2e151abf872ff Mon Sep 17 00:00:00 2001 From: bruno-f-cruz <7049351+bruno-f-cruz@users.noreply.github.com> Date: Thu, 9 May 2024 08:44:56 -0700 Subject: [PATCH 2/4] Deprecate function --- harp/reader.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/harp/reader.py b/harp/reader.py index a222c3a..f7e9eba 100644 --- a/harp/reader.py +++ b/harp/reader.py @@ -1,4 +1,6 @@ import os +import requests +from deprecated import deprecated from math import log2 from os import PathLike from pathlib import Path @@ -10,10 +12,10 @@ from typing import Any, BinaryIO, Callable, Iterable, Mapping, Optional, Protocol, Union from collections import UserDict from pandas._typing import Axes +from harp import __version__ from harp.model import BitMask, GroupMask, Model, PayloadMember, Register from harp.io import MessageType, read from harp.schema import read_schema -import requests @dataclass @@ -335,7 +337,7 @@ def parser(df: DataFrame): reader = partial(reader, columns=[name]) return RegisterReader(register, reader) - +@deprecated("This function is deprecated. Use DeviceReader.from_file, DeviceReader.from_url, DeviceReader.from_str, and DeviceReader.from_model instead.") def create_reader( device: Union[str, PathLike, Model], include_common_registers: bool = True, From 2e51fc4748247c2aae218dd1b9f945af11b737b5 Mon Sep 17 00:00:00 2001 From: bruno-f-cruz <7049351+bruno-f-cruz@users.noreply.github.com> Date: Thu, 9 May 2024 08:55:49 -0700 Subject: [PATCH 3/4] Document methods --- harp/reader.py | 173 ++++++++++++++++++++++++++++++++++++++++++------- 1 file changed, 149 insertions(+), 24 deletions(-) diff --git a/harp/reader.py b/harp/reader.py index f7e9eba..91726a0 100644 --- a/harp/reader.py +++ b/harp/reader.py @@ -80,11 +80,37 @@ def __getattr__(self, __name: str) -> RegisterReader: @staticmethod def from_file( - filepath: PathLike, - base_path: Optional[PathLike] = None, - include_common_registers: bool = True, - epoch: Optional[datetime] = None, - keep_type: bool = False) -> "DeviceReader": + filepath: PathLike, + base_path: Optional[PathLike] = None, + include_common_registers: bool = True, + epoch: Optional[datetime] = None, + keep_type: bool = False, + ) -> "DeviceReader": + """Creates a device reader object from the specified schema yml file. + + Parameters + ---------- + filepath + A path to the device yml schema describing the device. + base_path + The path to attempt to resolve the location of data files. + include_common_registers + Specifies whether to include the set of Harp common registers in the + parsed device schema object. If a parsed device schema object is provided, + this parameter is ignored. + epoch + The default reference datetime at which time zero begins. If specified, + the data frames returned by each register reader will have a datetime index. + keep_type + Specifies whether to include a column with the message type by default. + + Returns + ------- + A device reader object which can be used to read binary data for each + register or to access metadata about each register. Individual registers + can be accessed using dot notation using the name of the register as the + key. + """ device = read_schema(filepath, include_common_registers) if base_path is None: @@ -103,12 +129,39 @@ def from_file( @staticmethod def from_url( - url: str, - base_path: Optional[PathLike] = None, - include_common_registers: bool = True, - epoch: Optional[datetime] = None, - keep_type: bool = False, - timeout: int = 5) -> "DeviceReader": + url: str, + base_path: Optional[PathLike] = None, + include_common_registers: bool = True, + epoch: Optional[datetime] = None, + keep_type: bool = False, + timeout: int = 5, + ) -> "DeviceReader": + """Creates a device reader object from a url pointing to a device.yml file. + + Parameters + ---------- + url + The url pointing to the device.yml schema describing the device. + base_path + The path to attempt to resolve the location of data files. + include_common_registers + Specifies whether to include the set of Harp common registers in the + parsed device schema object. If a parsed device schema object is provided, + this parameter is ignored. + epoch + The default reference datetime at which time zero begins. If specified, + the data frames returned by each register reader will have a datetime index. + keep_type + Specifies whether to include a column with the message type by default. + timeout + The number of seconds to wait for the server to send data before giving up. + Returns + ------- + A device reader object which can be used to read binary data for each + register or to access metadata about each register. Individual registers + can be accessed using dot notation using the name of the register as the + key. + """ response = requests.get(url, timeout=timeout) text = response.text @@ -129,11 +182,37 @@ def from_url( @staticmethod def from_str( - schema: str, - base_path: Optional[PathLike] = None, - include_common_registers: bool = True, - epoch: Optional[datetime] = None, - keep_type: bool = False) -> "DeviceReader": + schema: str, + base_path: Optional[PathLike] = None, + include_common_registers: bool = True, + epoch: Optional[datetime] = None, + keep_type: bool = False, + ) -> "DeviceReader": + """Creates a device reader object from a string containing a device.yml schema. + + Parameters + ---------- + schema + The string containing the device.yml schema describing the device. + base_path + The path to attempt to resolve the location of data files. + include_common_registers + Specifies whether to include the set of Harp common registers in the + parsed device schema object. If a parsed device schema object is provided, + this parameter is ignored. + epoch + The default reference datetime at which time zero begins. If specified, + the data frames returned by each register reader will have a datetime index. + keep_type + Specifies whether to include a column with the message type by default. + + Returns + ------- + A device reader object which can be used to read binary data for each + register or to access metadata about each register. Individual registers + can be accessed using dot notation using the name of the register as the + key. + """ device = read_schema(schema, include_common_registers) if base_path is None: @@ -151,10 +230,32 @@ def from_str( @staticmethod def from_model( - model: Model, - base_path: Optional[PathLike] = None, - epoch: Optional[datetime] = None, - keep_type: bool = False) -> "DeviceReader": + model: Model, + base_path: Optional[PathLike] = None, + epoch: Optional[datetime] = None, + keep_type: bool = False, + ) -> "DeviceReader": + """Creates a device reader object from a parsed device schema object. + + Parameters + ---------- + model + The parsed device schema object describing the device. + base_path + The path to attempt to resolve the location of data files. + epoch + The default reference datetime at which time zero begins. If specified, + the data frames returned by each register reader will have a datetime index. + keep_type + Specifies whether to include a column with the message type by default. + + Returns + ------- + A device reader object which can be used to read binary data for each + register or to access metadata about each register. Individual registers + can be accessed using dot notation using the name of the register as the + key. + """ if base_path is None: base_path = Path(model.device).absolute().resolve() @@ -171,10 +272,34 @@ def from_model( @staticmethod def from_dataset( - dataset: PathLike, - include_common_registers: bool = True, - epoch: Optional[datetime] = None, - keep_type: bool = False) -> "DeviceReader": + dataset: PathLike, + include_common_registers: bool = True, + epoch: Optional[datetime] = None, + keep_type: bool = False, + ) -> "DeviceReader": + """Creates a device reader object from the specified dataset folder. + + Parameters + ---------- + dataset + A path to the dataset folder containing a device.yml schema describing the device. + include_common_registers + Specifies whether to include the set of Harp common registers in the + parsed device schema object. If a parsed device schema object is provided, + this parameter is ignored. + epoch + The default reference datetime at which time zero begins. If specified, + the data frames returned by each register reader will have a datetime index. + keep_type + Specifies whether to include a column with the message type by default. + + Returns + ------- + A device reader object which can be used to read binary data for each + register or to access metadata about each register. Individual registers + can be accessed using dot notation using the name of the register as the + key. + """ path = Path(dataset).absolute().resolve() is_dir = os.path.isdir(path) From 24b89773fdeace6fe4fa9ff01292c620eddb9b31 Mon Sep 17 00:00:00 2001 From: bruno-f-cruz <7049351+bruno-f-cruz@users.noreply.github.com> Date: Thu, 9 May 2024 08:55:57 -0700 Subject: [PATCH 4/4] Linting --- harp/io.py | 3 ++- harp/model.py | 12 +++--------- harp/reader.py | 34 +++++++++++++++++++++------------- harp/schema.py | 6 ++++-- 4 files changed, 30 insertions(+), 25 deletions(-) diff --git a/harp/io.py b/harp/io.py index 6c9630b..a37270e 100644 --- a/harp/io.py +++ b/harp/io.py @@ -2,9 +2,10 @@ from enum import IntEnum from os import PathLike from typing import Any, BinaryIO, Optional, Union -from pandas._typing import Axes + import numpy as np import pandas as pd +from pandas._typing import Axes REFERENCE_EPOCH = datetime(1904, 1, 1) """The reference epoch for UTC harp time.""" diff --git a/harp/model.py b/harp/model.py index 7c66f1d..f28a7be 100644 --- a/harp/model.py +++ b/harp/model.py @@ -6,16 +6,10 @@ from enum import Enum from typing import Dict, List, Optional, Union -from typing_extensions import Annotated -from pydantic import ( - BaseModel, - BeforeValidator, - ConfigDict, - Field, - RootModel, - field_serializer, -) +from pydantic import (BaseModel, BeforeValidator, ConfigDict, Field, RootModel, + field_serializer) +from typing_extensions import Annotated class PayloadType(str, Enum): diff --git a/harp/reader.py b/harp/reader.py index 91726a0..8f828f3 100644 --- a/harp/reader.py +++ b/harp/reader.py @@ -1,20 +1,23 @@ import os -import requests -from deprecated import deprecated +from collections import UserDict +from dataclasses import dataclass +from datetime import datetime +from functools import partial from math import log2 from os import PathLike from pathlib import Path -from datetime import datetime -from functools import partial -from dataclasses import dataclass +from typing import (Any, BinaryIO, Callable, Iterable, Mapping, Optional, + Protocol, Union) + +import requests +from deprecated import deprecated from numpy import dtype from pandas import DataFrame, Series -from typing import Any, BinaryIO, Callable, Iterable, Mapping, Optional, Protocol, Union -from collections import UserDict from pandas._typing import Axes + from harp import __version__ -from harp.model import BitMask, GroupMask, Model, PayloadMember, Register from harp.io import MessageType, read +from harp.model import BitMask, GroupMask, Model, PayloadMember, Register from harp.schema import read_schema @@ -31,8 +34,7 @@ def __call__( file: Optional[Union[str, bytes, PathLike[Any], BinaryIO]] = None, epoch: Optional[datetime] = None, keep_type: bool = False, - ) -> DataFrame: - ... + ) -> DataFrame: ... class RegisterReader: @@ -310,9 +312,12 @@ def from_dataset( base_path=path, include_common_registers=include_common_registers, epoch=epoch, - keep_type=keep_type) + keep_type=keep_type, + ) else: - raise ValueError("The dataset must be a directory containing a device.yml file.") + raise ValueError( + "The dataset must be a directory containing a device.yml file." + ) def _compose_parser( @@ -462,7 +467,10 @@ def parser(df: DataFrame): reader = partial(reader, columns=[name]) return RegisterReader(register, reader) -@deprecated("This function is deprecated. Use DeviceReader.from_file, DeviceReader.from_url, DeviceReader.from_str, and DeviceReader.from_model instead.") + +@deprecated( + "This function is deprecated. Use DeviceReader.from_file, DeviceReader.from_url, DeviceReader.from_str, and DeviceReader.from_model instead." +) def create_reader( device: Union[str, PathLike, Model], include_common_registers: bool = True, diff --git a/harp/schema.py b/harp/schema.py index 1bb1b0a..0787cc6 100644 --- a/harp/schema.py +++ b/harp/schema.py @@ -1,8 +1,10 @@ +from importlib import resources from os import PathLike from typing import TextIO, Union -from harp.model import Model, Registers + from pydantic_yaml import parse_yaml_raw_as -from importlib import resources + +from harp.model import Model, Registers def _read_common_registers() -> Registers: