Add a development mode

README.md

@@ -5,6 +5,30 @@
 This is a Fastapi application interfacing with a postgres database. It is designed to be deployed behind 
 Nginx on a kubernetes cluster.
 
+## Development
+
+.env
+```shell
+uri=postgresql://...
+
+REDIRECT_URI=http://localhost:8000/security/callback
+
+OAUTH_AUTHORIZATION_URL=https://cilogon.org/authorize
+OAUTH_TOKEN_URL=https://cilogon.org/oauth2/token
+OAUTH_USERINFO_URL=https://cilogon.org/oauth2/userinfo
+
+OAUTH_CLIENT_ID=
+OAUTH_CLIENT_SECRET=
+
+SECRET_KEY=<AnyRandomHash>
+JWT_ENCRYPTION_ALGORITHM=HS256
+
+access_key=<S3_ACCESS_KEY>
+secret_key=<S3_SECRET_KEY>
+
+ENVIRONMENT=development # This turns off authentication when running locally
+```
+
 ## Creating a token
 
 Assuming you are running locally on localhost:8000

api/app.py

@@ -26,7 +26,7 @@
 from api.models.geometries import PolygonModel, PolygonRequestModel, PolygonResponseModel, CopyColumnRequest
 from api.models.source import Sources
 from api.query_parser import ParserException
-from api.routes.security import TokenData, get_groups
+from api.routes.security import has_access
 from api.routes.object import router as object_router
 from api.routes.ingest import router as ingest_router
 
@@ -140,11 +140,11 @@ async def patch_sub_sources(
         request: starlette.requests.Request,
         table_id: int,
         polygon_updates: PolygonRequestModel,
-        groups: list[int] = Depends(get_groups)
+        user_has_access: bool = Depends(has_access),
 ):
 
-    if 1 not in groups:
-        raise HTTPException(status_code=401, detail="User is not in admin group")
+    if not user_has_access:
+        raise HTTPException(status_code=401, detail="User does not have access to patch object")
 
     try:
         result = await patch_sources_sub_table(
@@ -172,11 +172,11 @@ async def patch_sub_sources(
         target_column: str,
         table_id: int,
         copy_column: CopyColumnRequest,
-        groups: list[int] = Depends(get_groups)
+        user_has_access: bool = Depends(has_access),
 ):
 
-    if 1 not in groups:
-        raise HTTPException(status_code=401, detail="User is not in admin group")
+    if not user_has_access:
+        raise HTTPException(status_code=401, detail="User does not have access to patch object")
 
     try:
         result = await db.patch_sources_sub_table_set_columns_equal(

api/models/ingest.py

@@ -27,7 +27,7 @@ class Get(Post):
     object_group_id: int
     created_on: datetime.datetime
     completed_on: Optional[datetime.datetime] = None
-    source: Sources
+    source: Optional[Sources] = None
 
 
 class Patch(Post):

api/routes/ingest.py

@@ -10,7 +10,7 @@
     get_engine,
     results_to_model
 )
-from api.routes.security import get_groups
+from api.routes.security import has_access
 import api.models.ingest as IngestProcessModel
 import api.models.object as Object
 from api.schemas import IngestProcess as IngestProcessSchema, ObjectGroup, Sources
@@ -24,7 +24,7 @@
 
 
 @router.get("", response_model=list[IngestProcessModel.Get])
-async def get_multiple_ingest_process(page: int = 0, page_size: int = 50, filter_query_params=Depends(get_filter_query_params), groups: list[str] = Depends(get_groups)):
+async def get_multiple_ingest_process(page: int = 0, page_size: int = 50, filter_query_params=Depends(get_filter_query_params)):
     """Get all ingestion processes"""
 
     engine = get_engine()
@@ -47,29 +47,32 @@ async def get_multiple_ingest_process(page: int = 0, page_size: int = 50, filter
 
 
 @router.get("/{id}", response_model=IngestProcessModel.Get)
-async def get_ingest_process(id: int, groups: list[str] = Depends(get_groups)):
+async def get_ingest_process(id: int):
     """Get a single object"""
 
     engine = get_engine()
     async_session = get_async_session(engine)
 
     async with async_session() as session:
 
-        select_stmt = select(IngestProcessSchema).where(and_(IngestProcessSchema.id == id))
+        select_stmt = select(IngestProcessSchema).where(and_(IngestProcessSchema.id == id))\
+            .options(joinedload(IngestProcessSchema.source).defer(Sources.rgeom).defer(Sources.web_geom))
 
         result = await session.scalar(select_stmt)
 
         if result is None:
             raise HTTPException(status_code=404, detail=f"IngestProcess with id ({id}) not found")
 
-        response = IngestProcessModel.Get(**result.__dict__)
-        return response
+        return result
 
 
 @router.post("", response_model=IngestProcessModel.Get)
-async def create_ingest_process(object: IngestProcessModel.Post, groups: list[str] = Depends(get_groups)):
+async def create_ingest_process(object: IngestProcessModel.Post, user_has_access: bool = Depends(has_access)):
     """Create/Register a new object"""
 
+    if not user_has_access:
+        raise HTTPException(status_code=403, detail="User does not have access to create an object")
+
     engine = get_engine()
     async_session = get_async_session(engine,  expire_on_commit=False)
 
@@ -79,17 +82,27 @@ async def create_ingest_process(object: IngestProcessModel.Post, groups: list[st
         object_group = await session.scalar(object_group_stmt)
 
         stmt = insert(IngestProcessSchema).values(**object.model_dump(), object_group_id=object_group.id).returning(IngestProcessSchema)
+
         server_object = await session.scalar(stmt)
 
+        server_object.source = await session.scalar(select(Sources).where(Sources.source_id == server_object.source_id))
+
         await session.commit()
 
     return server_object
 
 
 @router.patch("/{id}", response_model=IngestProcessModel.Get)
-async def patch_ingest_process(id: int, object: IngestProcessModel.Patch, groups: list[str] = Depends(get_groups)):
+async def patch_ingest_process(
+        id: int,
+        object: IngestProcessModel.Patch,
+        user_has_access: bool = Depends(has_access)
+):
     """Update a object"""
 
+    if not user_has_access:
+        raise HTTPException(status_code=403, detail="User does not have access to create an object")
+
     engine = get_engine()
     async_session = get_async_session(engine)
 
@@ -108,7 +121,7 @@ async def patch_ingest_process(id: int, object: IngestProcessModel.Patch, groups
 
 
 @router.get("/{id}/objects", response_model=list[Object.GetSecureURL])
-async def get_ingest_process_objects(id: int, groups: list[str] = Depends(get_groups)):
+async def get_ingest_process_objects(id: int):
     """Get all objects for an ingestion process"""
 
     engine = get_engine()

api/routes/object.py

@@ -8,7 +8,7 @@
     get_engine,
     results_to_model
 )
-from api.routes.security import get_groups
+from api.routes.security import has_access
 import api.models.object as Object
 import api.schemas as schemas
 from api.query_parser import get_filter_query_params, QueryParser
@@ -21,7 +21,7 @@
 
 
 @router.get("", response_model=list[Object.Get])
-async def get_objects(page: int = 0, page_size: int = 50, filter_query_params=Depends(get_filter_query_params), groups: list[str] = Depends(get_groups)):
+async def get_objects(page: int = 0, page_size: int = 50, filter_query_params=Depends(get_filter_query_params)):
     """Get all objects"""
 
     engine = get_engine()
@@ -53,7 +53,7 @@ async def get_objects(page: int = 0, page_size: int = 50, filter_query_params=De
 
 
 @router.get("/{id}", response_model=Object.Get)
-async def get_object(id: int, groups: list[str] = Depends(get_groups)):
+async def get_object(id: int):
     """Get a single object"""
 
     engine = get_engine()
@@ -73,9 +73,12 @@ async def get_object(id: int, groups: list[str] = Depends(get_groups)):
 
 
 @router.post("", response_model=Object.Get)
-async def create_object(object: Object.Post, groups: list[str] = Depends(get_groups)):
+async def create_object(object: Object.Post, user_has_access: bool = Depends(has_access)):
     """Create/Register a new object"""
 
+    if not user_has_access:
+        raise HTTPException(status_code=403, detail="User does not have access to create object")
+
     engine = get_engine()
     async_session = get_async_session(engine)
 
@@ -90,9 +93,12 @@ async def create_object(object: Object.Post, groups: list[str] = Depends(get_gro
 
 
 @router.patch("/{id}", response_model=Object.Get)
-async def patch_object(id: int, object: Object.Patch, groups: list[str] = Depends(get_groups)):
+async def patch_object(id: int, object: Object.Patch, user_has_access: bool = Depends(has_access)):
     """Update a object"""
 
+    if not user_has_access:
+        raise HTTPException(status_code=403, detail="User does not have access to update object")
+
     engine = get_engine()
     async_session = get_async_session(engine)
 
@@ -111,9 +117,12 @@ async def patch_object(id: int, object: Object.Patch, groups: list[str] = Depend
 
 
 @router.delete("/{id}", response_model=Object.Get)
-async def delete_object(id: int, groups: list[str] = Depends(get_groups)):
+async def delete_object(id: int, has_access: bool = Depends(has_access)):
     """Delete a object"""
 
+    if not has_access:
+        raise HTTPException(status_code=403, detail="User does not have access to delete object")
+
     engine = get_engine()
     async_session = get_async_session(engine)
 

api/routes/security.py

@@ -176,6 +176,15 @@ async def get_groups(
     return groups
 
 
+async def has_access(groups: list[int] = Depends(get_groups)) -> bool:
+    """Check if the user has access to the group"""
+
+    if os.environ['ENVIRONMENT'] == 'development':
+        return True
+
+    return 1 in groups
+
+
 def create_access_token(data: dict, expires_delta: timedelta | None = None):
     """Create a JWT token"""
 

api/tests/main.py

@@ -162,10 +162,7 @@ def test_patch_source_tables(self, api_client):
 
     response = api_client.patch(
       f"/sources/{TEST_SOURCE_TABLE.source_id}/polygons",
-      json={TEST_SOURCE_TABLE.to_patch: id_temp_value},
-      headers={
-        "Authorization": f"Bearer {os.environ['ADMIN_TOKEN']}"
-      }
+      json={TEST_SOURCE_TABLE.to_patch: id_temp_value}
     )
 
     assert response.status_code == 204
@@ -202,10 +199,7 @@ def test_patch_source_tables_with_filter_in(self, api_client):
       json={
         TEST_SOURCE_TABLE.to_patch: id_temp_value
       },
-      params=TEST_SOURCE_TABLE.to_filter,
-      headers={
-        "Authorization": f"Bearer {os.environ['ADMIN_TOKEN']}"
-      }
+      params=TEST_SOURCE_TABLE.to_filter
     )
 
     assert response.status_code == 204
@@ -229,10 +223,7 @@ def test_patch_source_tables_with_filter(self, api_client):
     response = api_client.patch(
       f"/sources/{TEST_SOURCE_TABLE.source_id}/polygons",
       json=body,
-      params=params,
-      headers={
-        "Authorization": f"Bearer {os.environ['ADMIN_TOKEN']}"
-      }
+      params=params
     )
 
     assert response.status_code == 204
@@ -252,10 +243,7 @@ def test_patch_source_tables_with_filter_no_matches(self, api_client):
     response = api_client.patch(
       f"/sources/{TEST_SOURCE_TABLE.source_id}/polygons",
       json={TEST_SOURCE_TABLE.to_patch: id_temp_value},
-      params={"PTYPE": "eq.Qff", "orig_id": "eq.999999"},
-      headers={
-        "Authorization": f"Bearer {os.environ['ADMIN_TOKEN']}"
-      }
+      params={"PTYPE": "eq.Qff", "orig_id": "eq.999999"}
     )
 
     assert response.status_code == 400
@@ -282,7 +270,7 @@ def test_group_by_source_table(self, api_client):
     full_data = full_response.json()
 
     for row in full_data:
-      assert str(row["_pkid"]) in comparison_values[row["PTYPE"]]
+      assert str(row["_pkid"]) in comparison_values[row["PTYPE"]] or comparison_values[row["PTYPE"]] == "Multiple Values"
 
 
   def test_order_by_source_table(self, api_client):
@@ -309,9 +297,6 @@ def test_copy_column_values(self, api_client):
       f"/sources/{TEST_SOURCE_TABLE.source_id}/polygons",
       json={
         "descrip": test_value
-      },
-      headers={
-        "Authorization": f"Bearer {os.environ['ADMIN_TOKEN']}"
       }
     )
 
@@ -321,9 +306,6 @@ def test_copy_column_values(self, api_client):
       f"/sources/{TEST_SOURCE_TABLE.source_id}/polygons/comments",
       json={
         "source_column": "descrip"
-      },
-      headers={
-        "Authorization": f"Bearer {os.environ['ADMIN_TOKEN']}"
       }
     )