Merge pull request #23 from mlibrary/DWI-25-make-get-items-do-pagination

Dwi 25 make get items do pagination

aim/cli/main.py

@@ -3,6 +3,7 @@
 
 This hooks up the AIM CLI application. Nothing exciting happening here.
 """
+
 import typer
 import aim.cli.digifeeds as digifeeds
 

aim/digifeeds/database/crud.py

@@ -3,6 +3,7 @@
 
 Operations that act on the digifeeds database
 """
+
 from sqlalchemy.orm import Session
 from aim.digifeeds.database import schemas
 from aim.digifeeds.database import models
@@ -23,7 +24,12 @@ def get_item(db: Session, barcode: str):
     return db.query(models.Item).filter(models.Item.barcode == barcode).first()
 
 
-def get_items(db: Session, in_zephir: bool | None):
+def get_items_total(db: Session, in_zephir: bool | None):
+    query = get_items_query(db=db, in_zephir=in_zephir)
+    return query.count()
+
+
+def get_items(db: Session, in_zephir: bool | None, limit: int, offset: int):
     """
     Get Digifeed items from the database
 
@@ -34,26 +40,21 @@ def get_items(db: Session, in_zephir: bool | None):
     Returns:
         aim.digifeeds.database.models.Item: Item object
     """
+    query = get_items_query(db=db, in_zephir=in_zephir)
+    return query.offset(offset).limit(limit).all()
+
+
+def get_items_query(db: Session, in_zephir: bool | None):
+    query = db.query(models.Item)
     if in_zephir is True:
-        return (
-            db.query(models.Item)
-            .filter(
-                models.Item.statuses.any(
-                    models.ItemStatus.status_name == "in_zephir")
-            )
-            .all()
+        query = query.filter(
+            models.Item.statuses.any(models.ItemStatus.status_name == "in_zephir")
         )
     elif in_zephir is False:
-        return (
-            db.query(models.Item)
-            .filter(
-                ~models.Item.statuses.any(
-                    models.ItemStatus.status_name == "in_zephir")
-            )
-            .all()
+        query = query.filter(
+            ~models.Item.statuses.any(models.ItemStatus.status_name == "in_zephir")
         )
-
-    return db.query(models.Item).all()
+    return query
 
 
 def add_item(db: Session, item: schemas.ItemCreate):

aim/digifeeds/database/main.py

@@ -41,20 +41,27 @@ def get_db():  # pragma: no cover
 
 @app.get("/items/", response_model_by_alias=False, tags=["Digifeeds Database"])
 def get_items(
+    offset: int = Query(0, ge=0, description="Requested offset from the list of pages"),
+    limit: int = Query(50, ge=1, description="Requested number of items per page"),
     in_zephir: bool | None = Query(
         None, description="Filter for items that do or do not have metadata in Zephir"
     ),
     db: Session = Depends(get_db),
-) -> list[schemas.Item]:
+) -> schemas.PageOfItems:  # list[schemas.Item]:
     """
     Get the digifeeds items.
 
     These items can be filtered by whether or not their metadata is in Zephir or
     all of them can be fetched.
     """
 
-    db_items = crud.get_items(in_zephir=in_zephir, db=db)
-    return db_items
+    db_items = crud.get_items(in_zephir=in_zephir, db=db, offset=offset, limit=limit)
+    return {
+        "limit": limit,
+        "offset": offset,
+        "total": crud.get_items_total(in_zephir=in_zephir, db=db),
+        "items": db_items,
+    }
 
 
 @app.get(

aim/digifeeds/database/schemas.py

@@ -1,4 +1,5 @@
 """Digifeeds Pydantic Models"""
+
 from pydantic import BaseModel, Field, ConfigDict
 from datetime import datetime
 
@@ -39,6 +40,12 @@ class Item(ItemBase):
     )
 
 
+class PageOfItems(BaseModel):
+    items: list[Item]
+    limit: int = 10
+    offset: int = 0
+    total: int = 1
+
 
 class ItemCreate(ItemBase):
     pass
@@ -67,6 +74,7 @@ class Response400(Response):
         }
     )
 
+
 class Response404(Response):
     model_config = ConfigDict(
         json_schema_extra={

aim/digifeeds/db_client.py

@@ -68,5 +68,34 @@ def add_item_status(self, barcode: str, status: str):
             response.raise_for_status()
         return response.json()
 
+    def get_items(self, limit: int = 50, in_zephir: bool | None = None):
+        items = []
+        url = self._url("items")
+        params = {
+            "limit": limit,
+            "offset": 0,
+        }
+        if in_zephir is not None:
+            params["in_zephir"] = in_zephir
+
+        response = requests.get(url, params=params)
+        if response.status_code != 200:
+            response.raise_for_status()
+
+        first_page = response.json()
+        total = first_page["total"]
+        for item in first_page["items"]:
+            items.append(item)
+
+        for offset in list(range(limit, total, limit)):
+            params["offset"] = offset
+            response = requests.get(url, params=params)
+            if response.status_code != 200:
+                response.raise_for_status()
+            for item in response.json()["items"]:
+                items.append(item)
+
+        return items
+
     def _url(self, path) -> str:
         return f"{self.base_url}/{path}"

docs/conf.py

@@ -14,8 +14,15 @@
 # -- General configuration ---------------------------------------------------
 # https://www.sphinx-doc.org/en/master/usage/configuration.html#general-configuration
 
-extensions = ["sphinx.ext.napoleon", "sphinx.ext.viewcode", "sphinx.ext.autosummary",
-              "sphinx.ext.autodoc", 'myst_parser', 'sphinxcontrib.mermaid', "sphinx_toolbox.more_autodoc.autonamedtuple"]
+extensions = [
+    "sphinx.ext.napoleon",
+    "sphinx.ext.viewcode",
+    "sphinx.ext.autosummary",
+    "sphinx.ext.autodoc",
+    "myst_parser",
+    "sphinxcontrib.mermaid",
+    "sphinx_toolbox.more_autodoc.autonamedtuple",
+]
 autosummary_generate = True
 
 mermaid_d3_zoom = True
@@ -33,5 +40,5 @@
 html_theme_options = {
     "navigation_depth": 5,
     "collapse_navigation": False,
-    "titles_only": True
+    "titles_only": True,
 }

tests/conftest.py

@@ -10,8 +10,10 @@
 
 engine = create_engine(
     S.test_database,
-    connect_args={ "check_same_thread": False,},
-    poolclass=StaticPool
+    connect_args={
+        "check_same_thread": False,
+    },
+    poolclass=StaticPool,
 )
 
 TestingSessionLocal = sessionmaker(autocommit=False, autoflush=False, bind=engine)
@@ -25,10 +27,11 @@
 session.close()
 connection.close()
 
+
 # From: https://stackoverflow.com/questions/67255653/how-to-set-up-and-tear-down-a-database-between-tests-in-fastapi
 # These two event listeners are only needed for sqlite for proper
 # SAVEPOINT / nested transaction support. Other databases like postgres
-# don't need them. 
+# don't need them.
 # From: https://docs.sqlalchemy.org/en/14/dialects/sqlite.html#serializable-isolation-savepoints-transactional-ddl
 @sa.event.listens_for(engine, "connect")
 def do_connect(dbapi_connection, connection_record):
@@ -42,14 +45,14 @@ def do_begin(conn):
     # emit our own BEGIN
     conn.exec_driver_sql("BEGIN")
 
+
 # Handles rolling back the db after every test
 @pytest.fixture()
 def db_session(scope="module"):
     connection = engine.connect()
     transaction = connection.begin()
     session = TestingSessionLocal(bind=connection)
 
-
     # Begin a nested transaction (using SAVEPOINT).
     nested = connection.begin_nested()
 
@@ -68,6 +71,7 @@ def end_savepoint(session, transaction):
     transaction.rollback()
     connection.close()
 
+
 # A fixture for the fastapi test client which depends on the
 # previous session fixture. Instead of creating a new session in the
 # dependency override as before, it uses the one provided by the
@@ -79,4 +83,4 @@ def override_get_db():
 
     app.dependency_overrides[get_db] = override_get_db
     yield TestClient(app)
-    del app.dependency_overrides[get_db]
+    del app.dependency_overrides[get_db]

tests/digifeeds/database/test_crud.py

@@ -4,63 +4,71 @@
     add_item,
     get_status,
     get_statuses,
-    add_item_status
+    add_item_status,
+    get_items_total,
 )
 from aim.digifeeds.database.schemas import ItemCreate
 
+
 class TestCrud:
     def test_get_item(self, db_session):
         item = add_item(db=db_session, item=ItemCreate(barcode="valid_barcode"))
         barcode = item.barcode
         item_in_db = get_item(barcode=barcode, db=db_session)
-        assert(item_in_db.barcode) == "valid_barcode"
+        assert (item_in_db.barcode) == "valid_barcode"
 
     def test_get_item_that_does_not_exist(self, db_session):
         item_in_db = get_item(barcode="does not exist", db=db_session)
-        assert(item_in_db) is None
+        assert (item_in_db) is None
 
-    def test_get_items_all(self, db_session):
+    def test_get_items_and_total_any(self, db_session):
         item1 = add_item(db=db_session, item=ItemCreate(barcode="valid_barcode"))
         item2 = add_item(db=db_session, item=ItemCreate(barcode="valid_barcode2"))
         status = get_status(db=db_session, name="in_zephir")
-        add_item_status(db=db_session,item=item1, status=status)
-        items = get_items(db=db_session, in_zephir=None)
+        add_item_status(db=db_session, item=item1, status=status)
+        items = get_items(db=db_session, in_zephir=None, limit=2, offset=0)
+        count = get_items_total(db=db_session, in_zephir=None)
         db_session.refresh(item1)
         db_session.refresh(item2)
-        assert(items[0]) == item1
-        assert(items[1]) == item2
-
-    def test_get_items_in_zephir(self, db_session):
+        assert (items[0]) == item1
+        assert (items[1]) == item2
+        assert (count) == 2
+
+    def test_get_items_and_total_in_zephir(self, db_session):
         item1 = add_item(db=db_session, item=ItemCreate(barcode="valid_barcode"))
         item2 = add_item(db=db_session, item=ItemCreate(barcode="valid_barcode2"))
         status = get_status(db=db_session, name="in_zephir")
-        add_item_status(db=db_session,item=item1, status=status)
-        items = get_items(db=db_session, in_zephir=True)
+        add_item_status(db=db_session, item=item1, status=status)
+        items = get_items(db=db_session, in_zephir=True, limit=2, offset=0)
+        count = get_items_total(db=db_session, in_zephir=True)
         db_session.refresh(item1)
         db_session.refresh(item2)
-        assert(len(items)) == 1
-        assert(items[0]) == item1
+        assert (len(items)) == 1
+        assert (items[0]) == item1
+        assert count == 1
 
-    def test_get_items_not_in_zephir(self, db_session):
+    def test_get_items_and_total_not_in_zephir(self, db_session):
         item1 = add_item(db=db_session, item=ItemCreate(barcode="valid_barcode"))
         item2 = add_item(db=db_session, item=ItemCreate(barcode="valid_barcode2"))
         status = get_status(db=db_session, name="in_zephir")
-        add_item_status(db=db_session,item=item1, status=status)
-        items = get_items(db=db_session, in_zephir=False)
+        add_item_status(db=db_session, item=item1, status=status)
+        items = get_items(db=db_session, in_zephir=False, limit=2, offset=0)
+        count = get_items_total(db=db_session, in_zephir=False)
         db_session.refresh(item1)
         db_session.refresh(item2)
-        assert(len(items)) == 1
-        assert(items[0]) == item2
+        assert (len(items)) == 1
+        assert (items[0]) == item2
+        assert count == 1
 
     def test_get_status_that_exists(self, db_session):
         status = get_status(db=db_session, name="in_zephir")
-        assert(status.name) == "in_zephir"
+        assert (status.name) == "in_zephir"
 
     def test_get_status_that_does_not_exist(self, db_session):
         status = get_status(db=db_session, name="does_not_exist")
-        assert(status) is None 
+        assert (status) is None
 
     def test_get_statuses(self, db_session):
         statuses = get_statuses(db=db_session)
-        assert(len(statuses)) > 1
-        assert(statuses[0].name) == "in_zephir"
+        assert (len(statuses)) > 1
+        assert (statuses[0].name) == "in_zephir"