0.1 release

.github/workflows/python-test.yml

@@ -38,7 +38,7 @@ jobs:
              ubuntu-22.04,
              windows-2019,
              macos-13]
-        py-version: ["3.9", "3.10", "3.11", "3.12"]
+        py-version: ["3.11", "3.12"]
 
     steps:
       - uses: actions/checkout@v4

pyproject.toml

@@ -29,6 +29,11 @@ classifiers = [
 ]
 requires-python = ">=3.11"
 dependencies = [
+    "openpyxl",
+    "pandas",
+    "peewee",
+    "platformdirs",
+    "pydantic",
     "rdflib",
     "SPARQLWrapper",
     "structlog",

sentier_data_tools/__init__.py

@@ -2,11 +2,26 @@
 
 __all__ = (
     "__version__",
+    "Datapackage",
     "ProductIRI",
+    "Record",
     "UnitIRI",
+    "example_data_dir",
+    "reset_local_database",
 )
 
-__version__ = "0.0.1"
+__version__ = "0.1"
 
+from pathlib import Path
 
 from sentier_data_tools.iri import ProductIRI, UnitIRI
+from sentier_data_tools.local_data_store import (
+    Datapackage,
+    Record,
+    db,
+    initialize_local_database,
+    reset_local_database,
+)
+
+initialize_local_database(db)
+example_data_dir = Path(__file__).parent.resolve() / "example_data"

sentier_data_tools/datapackage.py

@@ -0,0 +1,76 @@
+import json
+import os
+from datetime import datetime, timezone
+from pathlib import Path
+from typing import List, Optional
+
+import pandas as pd
+
+from sentier_data_tools.datapackage_validation import Contributors, DatapackageMetadata
+from sentier_data_tools.licenses import LICENSES
+
+
+class Datapackage:
+    def __init__(
+        self,
+        *,
+        name: str,
+        description: str,
+        contributors: list,
+        homepage: Optional[str] = None,
+        created: Optional[datetime] = None,
+        version: str = "1",
+        licenses: Optional[list] = None,
+    ):
+        self.name = name
+        self.description = description
+        self.contributors = contributors
+        self.homepage = homepage
+        self.created = created or datetime.now(timezone.utc).isoformat()
+        self.licenses = licenses or [LICENSES["CC-BY-4.0"]]
+        self.version = version
+        self.resources = {}
+
+        for contributor in contributors:
+            Contributors(**contributor)
+        DatapackageMetadata(
+            name=self.name,
+            description=self.description,
+            homepage=self.homepage,
+            created=self.created,
+            version=self.version,
+            licenses=self.licenses,
+        )
+
+    def metadata(self) -> dict:
+        data = {
+            "name": self.name,
+            "description": self.description,
+            "contributors": self.contributors,
+            "created": self.created.isoformat()
+            if isinstance(self.created, datetime)
+            else self.created,
+            "version": self.version,
+            "licenses": self.licenses,
+        }
+        if self.homepage:
+            data["homepage"] = self.homepage
+        return data
+
+    def add_resource(
+        self, dataframe: pd.DataFrame, units: List[str], logs: Optional[list] = None
+    ) -> None:
+        pass
+
+    def to_json(self, filepath: Path) -> Path:
+        if not isinstance(filepath, Path):
+            filepath = Path(filepath)
+        if filepath.suffix.lower() != ".json":
+            filepath = filepath.parent / f"{filepath.name}.json"
+        if not os.access(filepath.parent, os.W_OK):
+            raise OSError(f"Can't write to directory {filepath.parent}")
+
+        with open(filepath, "w", encoding="utf-8") as f:
+            json.dump(self.metadata() | self.data(), f, indent=2, ensure_ascii=False)
+
+        return filepath

sentier_data_tools/datapackage_validation.py

@@ -0,0 +1,23 @@
+from datetime import datetime
+from typing import Dict, List, Optional
+
+from pydantic import BaseModel
+
+
+class Contributors(BaseModel):
+    title: str
+    role: str
+    path: str
+
+
+class DatapackageMetadata(BaseModel):
+    """Validate given Datapackage metadata.
+
+    Pydantic gives us nice error messages for free."""
+
+    name: str
+    description: str
+    homepage: Optional[str] = None
+    created: Optional[datetime] = datetime.now()
+    version: str
+    licenses: List[Dict[str, str]]

sentier_data_tools/iri.py

@@ -1,6 +1,6 @@
 from typing import List, Optional, Union
-from rdflib import URIRef, Literal, Graph
 
+from rdflib import Graph, Literal, URIRef
 from SPARQLWrapper import JSON, SPARQLWrapper
 
 from sentier_data_tools.logs import stdout_feedback_logger as logger
@@ -9,10 +9,12 @@
 
 
 def convert_json_object(obj: dict) -> Union[URIRef, Literal]:
-    if obj['type'] == 'literal':
-        return Literal(obj['value'], lang=obj.get("xml:lang"), datatype=obj.get('datatype'))
+    if obj["type"] == "literal":
+        return Literal(
+            obj["value"], lang=obj.get("xml:lang"), datatype=obj.get("datatype")
+        )
     else:
-        return URIRef(obj['value'])
+        return URIRef(obj["value"])
 
 
 class VocabIRI(URIRef):
@@ -46,17 +48,23 @@ def triples(self, subject: bool = True, limit: Optional[int] = 25) -> List[tuple
         logger.info(f"Retrieved {len(results)} triples from {VOCAB_FUSEKI}")
 
         if subject:
-            return [(
-                URIRef(str(self)),
-                convert_json_object(line['p']),
-                convert_json_object(line['o']),
-            ) for line in results]
+            return [
+                (
+                    URIRef(str(self)),
+                    convert_json_object(line["p"]),
+                    convert_json_object(line["o"]),
+                )
+                for line in results
+            ]
         else:
-            return [(
-                convert_json_object(line['s']),
-                convert_json_object(line['p']),
-                URIRef(str(self)),
-            ) for line in results]
+            return [
+                (
+                    convert_json_object(line["s"]),
+                    convert_json_object(line["p"]),
+                    URIRef(str(self)),
+                )
+                for line in results
+            ]
 
     def graph(self, subject: bool = True) -> Graph:
         """Return an `rdflib` graph of the data from the sentier.dev vocabulary for this IRI"""