From 7fb779c30b4ea99f4b4c74954c6c242e42e058c5 Mon Sep 17 00:00:00 2001 From: naglepuff Date: Thu, 10 Oct 2024 11:32:34 -0400 Subject: [PATCH 01/14] WIP Upgrade fastapi, pydantic --- nmdc_server/api.py | 4 +- nmdc_server/bulk_download_schema.py | 5 +- nmdc_server/config.py | 7 +- nmdc_server/data_object_filters.py | 4 +- nmdc_server/ingest/biosample.py | 14 +- nmdc_server/ingest/omics_processing.py | 8 +- nmdc_server/ingest/study.py | 5 +- nmdc_server/query.py | 25 +-- nmdc_server/schemas.py | 298 ++++++++++++------------- nmdc_server/schemas_submission.py | 56 ++--- pyproject.toml | 182 ++++++++++++--- tests/test_auth.py | 4 +- tests/test_download.py | 16 +- tests/test_submission.py | 2 +- tox.ini | 4 +- 15 files changed, 373 insertions(+), 261 deletions(-) diff --git a/nmdc_server/api.py b/nmdc_server/api.py index b8948add..131ced5e 100644 --- a/nmdc_server/api.py +++ b/nmdc_server/api.py @@ -47,8 +47,8 @@ async def get_version() -> schemas.VersionInfo: # get the current user information @router.get("/me", tags=["user"], name="Return the current user name") -async def me(user: User = Depends(get_current_user)) -> Optional[User]: - return user +async def me(user: User = Depends(get_current_user)) -> Optional[schemas.User]: + return schemas.User(**user) # autocomplete search diff --git a/nmdc_server/bulk_download_schema.py b/nmdc_server/bulk_download_schema.py index 93d0217a..f1870c76 100644 --- a/nmdc_server/bulk_download_schema.py +++ b/nmdc_server/bulk_download_schema.py @@ -5,6 +5,7 @@ from nmdc_server.data_object_filters import DataObjectFilter from nmdc_server.query import ConditionSchema from nmdc_server.schemas import FileDownloadMetadata +from pydantic import ConfigDict # schemas related to bulk download endpoints extracted @@ -17,9 +18,7 @@ class BulkDownloadBase(FileDownloadMetadata): class BulkDownload(BulkDownloadBase): id: UUID created: datetime - - class Config: - orm_mode = True + model_config = ConfigDict(from_attributes=True) class BulkDownloadCreate(BulkDownloadBase): diff --git a/nmdc_server/config.py b/nmdc_server/config.py index 2df2c9ad..db19af59 100644 --- a/nmdc_server/config.py +++ b/nmdc_server/config.py @@ -1,7 +1,6 @@ import os from typing import Optional - -from pydantic import BaseSettings +from pydantic_settings import BaseSettings, SettingsConfigDict class Settings(BaseSettings): @@ -96,9 +95,7 @@ def current_db_uri(self) -> str: return self.testing_database_uri return self.database_uri - class Config: - env_prefix = "nmdc_" - env_file = os.getenv("DOTENV_PATH", ".env") + model_config = SettingsConfigDict(env_prefix="nmdc_", env_file=os.getenv("DOTENV_PATH", ".env")) settings = Settings() diff --git a/nmdc_server/data_object_filters.py b/nmdc_server/data_object_filters.py index 8b7e1992..fc5d8482 100644 --- a/nmdc_server/data_object_filters.py +++ b/nmdc_server/data_object_filters.py @@ -84,5 +84,5 @@ def output_association(self): class DataObjectFilter(BaseModel): - workflow: Optional[WorkflowActivityTypeEnum] - file_type: Optional[str] + workflow: Optional[WorkflowActivityTypeEnum] = None + file_type: Optional[str] = None diff --git a/nmdc_server/ingest/biosample.py b/nmdc_server/ingest/biosample.py index a570621f..f74f69d3 100644 --- a/nmdc_server/ingest/biosample.py +++ b/nmdc_server/ingest/biosample.py @@ -3,7 +3,7 @@ from datetime import datetime from typing import Any, Dict -from pydantic import root_validator, validator +from pydantic import field_validator, model_validator, validator from pymongo.cursor import Cursor from sqlalchemy.orm import Session @@ -20,7 +20,8 @@ class Biosample(BiosampleCreate): _extract_value = validator("*", pre=True, allow_reuse=True)(extract_value) - @root_validator(pre=True) + @model_validator(mode="before") + @classmethod def extract_extras(cls, values): if "lat_lon" in values: if "latitude" in values["lat_lon"] and "longitude" in values["lat_lon"]: @@ -32,20 +33,23 @@ def extract_extras(cls, values): values["longitude"] = float(lon) return extract_extras(cls, values) - @validator("depth", pre=True) + @field_validator("depth", mode="before") + @classmethod def normalize_depth(cls, value): value = extract_value(value) if isinstance(value, str): return float(value.split(" ")[0]) return value - @validator("add_date", "mod_date", pre=True) + @field_validator("add_date", "mod_date", mode="before") + @classmethod def coerce_date(cls, v): if isinstance(v, str) and date_fmt.match(v): return datetime.strptime(v, "%d-%b-%y %I.%M.%S.%f000 %p").isoformat() return v - @validator("collection_date", pre=True) + @field_validator("collection_date", mode="before") + @classmethod def coerce_collection_date(cls, value): # { "has_raw_value": ... } raw_value = value["has_raw_value"] diff --git a/nmdc_server/ingest/omics_processing.py b/nmdc_server/ingest/omics_processing.py index 7fe82c7c..e8bd510a 100644 --- a/nmdc_server/ingest/omics_processing.py +++ b/nmdc_server/ingest/omics_processing.py @@ -3,7 +3,7 @@ from datetime import datetime from typing import Any, Dict, Optional -from pydantic import root_validator, validator +from pydantic import field_validator, model_validator, validator from pymongo.collection import Collection from pymongo.cursor import Cursor from pymongo.database import Database @@ -35,11 +35,13 @@ class OmicsProcessing(OmicsProcessingCreate): _extract_value = validator("*", pre=True, allow_reuse=True)(extract_value) - @root_validator(pre=True) + @model_validator(mode="before") + @classmethod def extract_extras(cls, values): return extract_extras(cls, values) - @validator("add_date", "mod_date", pre=True) + @field_validator("add_date", "mod_date", mode="before") + @classmethod def coerce_date(cls, v): if isinstance(v, str) and date_fmt.match(v): return datetime.strptime(v, "%d-%b-%y %I.%M.%S.%f000 %p").isoformat() diff --git a/nmdc_server/ingest/study.py b/nmdc_server/ingest/study.py index b0ab065e..3c01a5d4 100644 --- a/nmdc_server/ingest/study.py +++ b/nmdc_server/ingest/study.py @@ -2,7 +2,7 @@ from typing import Optional import requests -from pydantic import root_validator, validator +from pydantic import model_validator, validator from pymongo.cursor import Cursor from sqlalchemy.orm import Session @@ -41,7 +41,8 @@ def get_or_create_pi(db: Session, name: str, url: Optional[str], orcid: Optional class Study(StudyCreate): _extract_value = validator("*", pre=True, allow_reuse=True)(extract_value) - @root_validator(pre=True) + @model_validator(mode="before") + @classmethod def extract_extras(cls, values): return extract_extras(cls, values) diff --git a/nmdc_server/query.py b/nmdc_server/query.py index 1638dbb4..6de7063f 100644 --- a/nmdc_server/query.py +++ b/nmdc_server/query.py @@ -9,7 +9,7 @@ from itertools import groupby from typing import Any, Dict, Iterator, List, Optional, Tuple, Union -from pydantic import BaseModel, Field, PositiveInt +from pydantic import ConfigDict, BaseModel, Field, PositiveInt from sqlalchemy import ARRAY, Column, and_, cast, desc, func, inspect, or_ from sqlalchemy.orm import Query, Session, aliased, with_expression from sqlalchemy.orm.util import AliasedClass @@ -106,11 +106,11 @@ class Operation(Enum): class GoldTreeValue(BaseModel): - ecosystem: Optional[str] - ecosystem_category: Optional[str] - ecosystem_type: Optional[str] - ecosystem_subtype: Optional[str] - specific_ecosystem: Optional[str] + ecosystem: Optional[str] = None + ecosystem_category: Optional[str] = None + ecosystem_type: Optional[str] = None + ecosystem_subtype: Optional[str] = None + specific_ecosystem: Optional[str] = None ConditionValue = Union[schemas.AnnotationValue, RangeValue, List[GoldTreeValue]] @@ -902,8 +902,7 @@ class SearchQuery(BaseModel): class ConditionResultSchema(SimpleConditionSchema): - class Config: - orm_mode = True + model_config = ConfigDict(from_attributes=True) class FacetQuery(SearchQuery): @@ -915,14 +914,14 @@ class BiosampleSearchQuery(SearchQuery): class BinnedRangeFacetQuery(FacetQuery): - minimum: Optional[NumericValue] - maximum: Optional[NumericValue] + minimum: Optional[NumericValue] = None + maximum: Optional[NumericValue] = None num_bins: PositiveInt class BinnedDateFacetQuery(FacetQuery): - minimum: Optional[datetime] - maximum: Optional[datetime] + minimum: Optional[datetime] = None + maximum: Optional[datetime] = None resolution: DateBinResolution @@ -931,7 +930,7 @@ class BinnedDateFacetQuery(FacetQuery): class StudySearchResponse(BaseSearchResponse): results: List[schemas.Study] - total: Optional[int] + total: Optional[int] = None class OmicsProcessingSearchResponse(BaseSearchResponse): diff --git a/nmdc_server/schemas.py b/nmdc_server/schemas.py index 3edc65a5..40b5fcea 100644 --- a/nmdc_server/schemas.py +++ b/nmdc_server/schemas.py @@ -15,7 +15,7 @@ from uuid import UUID from pint import Unit -from pydantic import BaseModel, Field, validator +from pydantic import BaseModel, ConfigDict, Field, validator from sqlalchemy import BigInteger, Column, DateTime, Float, Integer, LargeBinary, String from sqlalchemy.dialects.postgresql.json import JSONB @@ -35,7 +35,7 @@ class ErrorSchema(BaseModel): message: str = Field( ..., description="Human-readable error message.", - example="Something went wrong.", + examples=["Something went wrong."], ) @@ -44,7 +44,7 @@ class InternalErrorSchema(ErrorSchema): ..., description="Unique identifier for the error that occurred. Provide this to system " "administrators if you are reporting an error.", - example="dd4c4fa3-8d22-4768-8b0d-0923140d9f8a", + examples=["dd4c4fa3-8d22-4768-8b0d-0923140d9f8a"], ) @@ -77,9 +77,7 @@ class EnvoTerm(BaseModel): label: str url: str data: Dict[str, Any] - - class Config: - orm_mode = True + model_config = ConfigDict(from_attributes=True) class AnnotatedBase(BaseModel): @@ -125,10 +123,10 @@ def from_unit(cls, unit: Optional[Unit]) -> Optional["UnitInfo"]: class AttributeSummary(BaseModel): count: int - min: Optional[Union[float, datetime]] - max: Optional[Union[float, datetime]] + min: Optional[Union[float, datetime]] = None + max: Optional[Union[float, datetime]] = None type: AttributeType - units: Optional[UnitInfo] + units: Optional[UnitInfo] = None class TableSummary(BaseModel): @@ -169,25 +167,21 @@ class AggregationSummary(BaseModel): class EnvironmentSankeyAggregation(BaseModel): count: int - ecosystem: Optional[str] - ecosystem_category: Optional[str] - ecosystem_type: Optional[str] - ecosystem_subtype: Optional[str] - specific_ecosystem: Optional[str] - - class Config: - orm_mode = True + ecosystem: Optional[str] = None + ecosystem_category: Optional[str] = None + ecosystem_type: Optional[str] = None + ecosystem_subtype: Optional[str] = None + specific_ecosystem: Optional[str] = None + model_config = ConfigDict(from_attributes=True) class EnvironmentGeospatialAggregation(BaseModel): count: int - latitude: Optional[float] - longitude: Optional[float] - ecosystem: Optional[str] - ecosystem_category: Optional[str] - - class Config: - orm_mode = True + latitude: Optional[float] = None + longitude: Optional[float] = None + ecosystem: Optional[str] = None + ecosystem_category: Optional[str] = None + model_config = ConfigDict(from_attributes=True) class DataObjectAggregationNode(BaseModel): @@ -205,13 +199,11 @@ class DataObjectAggregationElement(DataObjectAggregationNode): class OrcidPerson(BaseModel): """https://microbiomedata.github.io/nmdc-schema/PersonValue/""" - name: Optional[str] - email: Optional[str] - orcid: Optional[str] - profile_image_url: Optional[str] - - class Config: - orm_mode = True + name: Optional[str] = None + email: Optional[str] = None + orcid: Optional[str] = None + profile_image_url: Optional[str] = None + model_config = ConfigDict(from_attributes=True) class CreditAssociation(BaseModel): @@ -226,9 +218,7 @@ class DOIInfo(BaseModel): info: dict doi_category: models.DOIType doi_provider: str - - class Config: - orm_mode = True + model_config = ConfigDict(from_attributes=True) class StudyBase(AnnotatedBase): @@ -236,17 +226,19 @@ class StudyBase(AnnotatedBase): gold_name: str = "" gold_description: str = "" scientific_objective: str = "" - add_date: Optional[DateType] - mod_date: Optional[DateType] - has_credit_associations: Optional[List[CreditAssociation]] - relevant_protocols: Optional[List[str]] - funding_sources: Optional[List[str]] - gold_study_identifiers: Optional[List[str]] - homepage_website: Optional[List[str]] - part_of: Optional[List[str]] - study_category: Optional[str] + add_date: Optional[DateType] = None + mod_date: Optional[DateType] = None + has_credit_associations: Optional[List[CreditAssociation]] = None + relevant_protocols: Optional[List[str]] = None + funding_sources: Optional[List[str]] = None + gold_study_identifiers: Optional[List[str]] = None + homepage_website: Optional[List[str]] = None + part_of: Optional[List[str]] = None + study_category: Optional[str] = None children: Optional[List[Study]] = [] + # TODO[pydantic]: We couldn't refactor the `validator`, please replace it by `field_validator` manually. + # Check https://docs.pydantic.dev/dev-v2/migration/#changes-to-validators for more information. @validator("principal_investigator_websites", pre=True, each_item=True) def replace_websites(cls, study_website: Union[models.StudyWebsite, str]) -> str: if isinstance(study_website, str): @@ -255,57 +247,55 @@ def replace_websites(cls, study_website: Union[models.StudyWebsite, str]) -> str class StudyCreate(StudyBase): - principal_investigator_id: Optional[UUID] - image: Optional[bytes] + principal_investigator_id: Optional[UUID] = None + image: Optional[bytes] = None class OmicsCounts(BaseModel): type: str count: int + # TODO[pydantic]: We couldn't refactor the `validator`, please replace it by `field_validator` manually. + # Check https://docs.pydantic.dev/dev-v2/migration/#changes-to-validators for more information. @validator("count", pre=True, always=True) def insert_zero(cls, v): return v or 0 class Study(StudyBase): - open_in_gold: Optional[str] - principal_investigator: Optional[OrcidPerson] - principal_investigator_name: Optional[str] + open_in_gold: Optional[str] = None + principal_investigator: Optional[OrcidPerson] = None + principal_investigator_name: Optional[str] = None image_url: str principal_investigator_image_url: str - sample_count: Optional[int] - omics_counts: Optional[List[OmicsCounts]] - omics_processing_counts: Optional[List[OmicsCounts]] + sample_count: Optional[int] = None + omics_counts: Optional[List[OmicsCounts]] = None + omics_processing_counts: Optional[List[OmicsCounts]] = None doi_map: Dict[str, Any] = {} multiomics: int - - class Config: - orm_mode = True + model_config = ConfigDict(from_attributes=True) # biosample class BiosampleBase(AnnotatedBase): study_id: str - depth: Optional[float] - env_broad_scale_id: Optional[str] - env_local_scale_id: Optional[str] - env_medium_id: Optional[str] + depth: Optional[float] = None + env_broad_scale_id: Optional[str] = None + env_local_scale_id: Optional[str] = None + env_medium_id: Optional[str] = None # https://github.com/samuelcolvin/pydantic/issues/156 longitude: Optional[float] = Field(default=None, gt=-180, le=180) latitude: Optional[float] = Field(default=None, ge=-90, le=90) - add_date: Optional[DateType] - mod_date: Optional[DateType] - - collection_date: Optional[DateType] - ecosystem: Optional[str] - ecosystem_category: Optional[str] - ecosystem_type: Optional[str] - ecosystem_subtype: Optional[str] - specific_ecosystem: Optional[str] + add_date: Optional[DateType] = None + mod_date: Optional[DateType] = None - class Config: - orm_mode = True + collection_date: Optional[DateType] = None + ecosystem: Optional[str] = None + ecosystem_category: Optional[str] = None + ecosystem_type: Optional[str] = None + ecosystem_subtype: Optional[str] = None + specific_ecosystem: Optional[str] = None + model_config = ConfigDict(from_attributes=True) class BiosampleCreate(BiosampleBase): @@ -313,28 +303,26 @@ class BiosampleCreate(BiosampleBase): class Biosample(BiosampleBase): - open_in_gold: Optional[str] - env_broad_scale: Optional[EnvoTerm] - env_local_scale: Optional[EnvoTerm] - env_medium: Optional[EnvoTerm] + open_in_gold: Optional[str] = None + env_broad_scale: Optional[EnvoTerm] = None + env_local_scale: Optional[EnvoTerm] = None + env_medium: Optional[EnvoTerm] = None env_broad_scale_terms: List[str] = [] env_local_scale_terms: List[str] = [] env_medium_terms: List[str] = [] - emsl_biosample_identifiers: Optional[List[str]] + emsl_biosample_identifiers: Optional[List[str]] = None omics_processing: List["OmicsProcessing"] multiomics: int - - class Config: - orm_mode = True + model_config = ConfigDict(from_attributes=True) # omics_processing class OmicsProcessingBase(AnnotatedBase): - study_id: Optional[str] + study_id: Optional[str] = None biosample_inputs: list[BiosampleBase] = [] - add_date: Optional[DateType] - mod_date: Optional[DateType] + add_date: Optional[DateType] = None + mod_date: Optional[DateType] = None class OmicsProcessingCreate(OmicsProcessingBase): @@ -342,12 +330,14 @@ class OmicsProcessingCreate(OmicsProcessingBase): class OmicsProcessing(OmicsProcessingBase): - open_in_gold: Optional[str] + open_in_gold: Optional[str] = None biosample_ids: list[str] = [] omics_data: List["OmicsTypes"] outputs: List["DataObject"] + # TODO[pydantic]: We couldn't refactor the `validator`, please replace it by `field_validator` manually. + # Check https://docs.pydantic.dev/dev-v2/migration/#changes-to-validators for more information. @validator("biosample_ids") @classmethod def set_biosample_ids(cls, biosample_ids: list[str], values: dict[str, Any]) -> list[str]: @@ -358,8 +348,7 @@ def set_biosample_ids(cls, biosample_ids: list[str], values: dict[str, Any]) -> return biosample_ids - class Config: - orm_mode = True + model_config = ConfigDict(from_attributes=True) # data_object @@ -367,9 +356,9 @@ class DataObjectBase(BaseModel): id: str name: str description: str = "" - file_size_bytes: Optional[int] - md5_checksum: Optional[str] - url: Optional[str] + file_size_bytes: Optional[int] = None + md5_checksum: Optional[str] = None + url: Optional[str] = None downloads: int file_type: Optional[str] = None file_type_description: Optional[str] = None @@ -381,10 +370,10 @@ class DataObjectCreate(DataObjectBase): class DataObject(DataObjectBase): selected: Optional[bool] = None + model_config = ConfigDict(from_attributes=True) - class Config: - orm_mode = True - + # TODO[pydantic]: We couldn't refactor the `validator`, please replace it by `field_validator` manually. + # Check https://docs.pydantic.dev/dev-v2/migration/#changes-to-validators for more information. @validator("url") def replace_url(cls, url, values): id_str = quote(values["id"]) @@ -422,8 +411,7 @@ def file_type_match(f, file_type) -> bool: class GeneFunction(BaseModel): - class Config: - orm_mode = True + model_config = ConfigDict(from_attributes=True) id: str @@ -443,17 +431,15 @@ class PipelineStep(PipelineStepBase): # has_inputs: List[str] # has_outputs: List[str] outputs: List[DataObject] - - class Config: - orm_mode = True + model_config = ConfigDict(from_attributes=True) class ReadsQCBase(PipelineStepBase): type: str = WorkflowActivityTypeEnum.reads_qc.value - input_read_count: Optional[int] - input_read_bases: Optional[int] - output_read_count: Optional[int] - output_read_bases: Optional[int] + input_read_count: Optional[int] = None + input_read_bases: Optional[int] = None + output_read_count: Optional[int] = None + output_read_bases: Optional[int] = None class ReadsQC(PipelineStep): @@ -461,35 +447,35 @@ class ReadsQC(PipelineStep): class AssemblyBase(PipelineStepBase): - scaffolds: Optional[int] - contigs: Optional[int] - scaf_bp: Optional[int] - contig_bp: Optional[int] - scaf_n50: Optional[int] - scaf_l50: Optional[int] - ctg_n50: Optional[int] - ctg_l50: Optional[int] - scaf_n90: Optional[int] - scaf_l90: Optional[int] - ctg_n90: Optional[int] - ctg_l90: Optional[int] - scaf_max: Optional[int] - ctg_max: Optional[int] - scaf_n_gt50k: Optional[int] + scaffolds: Optional[int] = None + contigs: Optional[int] = None + scaf_bp: Optional[int] = None + contig_bp: Optional[int] = None + scaf_n50: Optional[int] = None + scaf_l50: Optional[int] = None + ctg_n50: Optional[int] = None + ctg_l50: Optional[int] = None + scaf_n90: Optional[int] = None + scaf_l90: Optional[int] = None + ctg_n90: Optional[int] = None + ctg_l90: Optional[int] = None + scaf_max: Optional[int] = None + ctg_max: Optional[int] = None + scaf_n_gt50k: Optional[int] = None # TODO: fix the data on ingest or make this optional on the schema - scaf_l_gt50k: Optional[int] - scaf_pct_gt50k: Optional[int] - num_input_reads: Optional[int] - num_aligned_reads: Optional[int] - scaf_logsum: Optional[float] - scaf_powsum: Optional[float] - ctg_logsum: Optional[float] - ctg_powsum: Optional[float] - asm_score: Optional[float] - gap_pct: Optional[float] - gc_avg: Optional[float] - gc_std: Optional[float] + scaf_l_gt50k: Optional[int] = None + scaf_pct_gt50k: Optional[int] = None + num_input_reads: Optional[int] = None + num_aligned_reads: Optional[int] = None + scaf_logsum: Optional[float] = None + scaf_powsum: Optional[float] = None + ctg_logsum: Optional[float] = None + ctg_powsum: Optional[float] = None + asm_score: Optional[float] = None + gap_pct: Optional[float] = None + gc_avg: Optional[float] = None + gc_std: Optional[float] = None class MetagenomeAssemblyBase(AssemblyBase): @@ -533,16 +519,16 @@ class MetaproteomicAnalysis(PipelineStep): class MAG(BaseModel): - bin_name: Optional[str] - number_of_contig: Optional[int] - completeness: Optional[float] - contamination: Optional[float] - gene_count: Optional[int] - bin_quality: Optional[str] - num_16s: Optional[int] - num_5s: Optional[int] - num_23s: Optional[int] - num_t_rna: Optional[int] + bin_name: Optional[str] = None + number_of_contig: Optional[int] = None + completeness: Optional[float] = None + contamination: Optional[float] = None + gene_count: Optional[int] = None + bin_quality: Optional[str] = None + num_16s: Optional[int] = None + num_5s: Optional[int] = None + num_23s: Optional[int] = None + num_t_rna: Optional[int] = None class MAGCreate(MAG): @@ -551,20 +537,20 @@ class MAGCreate(MAG): class MAGsAnalysisBase(PipelineStepBase): type: str = WorkflowActivityTypeEnum.mags_analysis.value - input_contig_num: Optional[int] - too_short_contig_num: Optional[int] - low_depth_contig_num: Optional[int] - unbinned_contig_num: Optional[int] - binned_contig_num: Optional[int] + input_contig_num: Optional[int] = None + too_short_contig_num: Optional[int] = None + low_depth_contig_num: Optional[int] = None + unbinned_contig_num: Optional[int] = None + binned_contig_num: Optional[int] = None class MAGsAnalysis(PipelineStep): type: str = WorkflowActivityTypeEnum.mags_analysis.value - input_contig_num: Optional[int] - too_short_contig_num: Optional[int] - low_depth_contig_num: Optional[int] - unbinned_contig_num: Optional[int] - binned_contig_num: Optional[int] + input_contig_num: Optional[int] = None + too_short_contig_num: Optional[int] = None + low_depth_contig_num: Optional[int] = None + unbinned_contig_num: Optional[int] = None + binned_contig_num: Optional[int] = None mags_list: List[MAG] @@ -664,9 +650,7 @@ class KeggTermListResponse(BaseModel): class KeggTermText(BaseModel): term: str text: str - - class Config: - orm_mode = True + model_config = ConfigDict(from_attributes=True) class KeggTermTextListResponse(BaseModel): @@ -679,20 +663,18 @@ class IngestArgumentSchema(BaseModel): class User(BaseModel): - id: Optional[UUID] + id: Optional[UUID] = None orcid: str name: str = "" - is_admin = False - - class Config: - orm_mode = True + is_admin: bool = False + model_config = ConfigDict(from_attributes=True) class LockOperationResult(BaseModel): success: bool message: str - locked_by: Optional[User] - lock_updated: Optional[datetime] + locked_by: Optional[User] = None + lock_updated: Optional[datetime] = None class VersionInfo(BaseModel): @@ -705,8 +687,4 @@ class VersionInfo(BaseModel): nmdc_server: str = __version__ nmdc_schema: str = version("nmdc-schema") nmdc_submission_schema: str = version("nmdc-submission-schema") - - class Config: - # In Pydantic V2, use `frozen=True` - # https://docs.pydantic.dev/2.8/concepts/models/#faux-immutability - allow_mutation = False + model_config = ConfigDict(frozen=False) diff --git a/nmdc_server/schemas_submission.py b/nmdc_server/schemas_submission.py index 04c95db4..6a17a162 100644 --- a/nmdc_server/schemas_submission.py +++ b/nmdc_server/schemas_submission.py @@ -3,7 +3,7 @@ from typing import Any, Dict, List, Optional from uuid import UUID -from pydantic import BaseModel, validator +from pydantic import ConfigDict, BaseModel, validator from nmdc_server import schemas from nmdc_server.models import SubmissionEditorRole @@ -13,7 +13,7 @@ class Contributor(BaseModel): name: str orcid: str roles: List[str] - permissionLevel: Optional[str] + permissionLevel: Optional[str] = None class StudyForm(BaseModel): @@ -21,9 +21,9 @@ class StudyForm(BaseModel): piName: str piEmail: str piOrcid: str - fundingSources: Optional[List[str]] + fundingSources: Optional[List[str]] = None linkOutWebpage: List[str] - studyDate: Optional[str] + studyDate: Optional[str] = None description: str notes: str contributors: List[Contributor] @@ -51,25 +51,25 @@ class NmcdAddress(BaseModel): class AddressForm(BaseModel): shipper: NmcdAddress - expectedShippingDate: Optional[datetime] + expectedShippingDate: Optional[datetime] = None shippingConditions: str sample: str description: str experimentalGoals: str randomization: str - usdaRegulated: Optional[bool] + usdaRegulated: Optional[bool] = None permitNumber: str biosafetyLevel: str - irbOrHipaa: Optional[bool] + irbOrHipaa: Optional[bool] = None comments: str class ContextForm(BaseModel): datasetDoi: str - dataGenerated: Optional[bool] - facilityGenerated: Optional[bool] + dataGenerated: Optional[bool] = None + facilityGenerated: Optional[bool] = None facilities: List[str] - award: Optional[str] + award: Optional[str] = None otherAward: str @@ -84,26 +84,26 @@ class MetadataSubmissionRecord(BaseModel): class PartialMetadataSubmissionRecord(BaseModel): - packageName: Optional[str] - contextForm: Optional[ContextForm] - addressForm: Optional[AddressForm] - templates: Optional[List[str]] - studyForm: Optional[StudyForm] - multiOmicsForm: Optional[MultiOmicsForm] - sampleData: Optional[Dict[str, List[Any]]] + packageName: Optional[str] = None + contextForm: Optional[ContextForm] = None + addressForm: Optional[AddressForm] = None + templates: Optional[List[str]] = None + studyForm: Optional[StudyForm] = None + multiOmicsForm: Optional[MultiOmicsForm] = None + sampleData: Optional[Dict[str, List[Any]]] = None class SubmissionMetadataSchemaCreate(BaseModel): metadata_submission: MetadataSubmissionRecord - status: Optional[str] - source_client: Optional[str] + status: Optional[str] = None + source_client: Optional[str] = None class SubmissionMetadataSchemaPatch(BaseModel): metadata_submission: PartialMetadataSubmissionRecord - status: Optional[str] + status: Optional[str] = None # Map of ORCID iD to permission level - permissions: Optional[Dict[str, str]] + permissions: Optional[Dict[str, str]] = None class SubmissionMetadataSchema(SubmissionMetadataSchemaCreate): @@ -113,16 +113,16 @@ class SubmissionMetadataSchema(SubmissionMetadataSchemaCreate): status: str author: schemas.User templates: List[str] - study_name: Optional[str] + study_name: Optional[str] = None - lock_updated: Optional[datetime] - locked_by: Optional[schemas.User] + lock_updated: Optional[datetime] = None + locked_by: Optional[schemas.User] = None - permission_level: Optional[str] - - class Config: - orm_mode = True + permission_level: Optional[str] = None + model_config = ConfigDict(from_attributes=True) + # TODO[pydantic]: We couldn't refactor the `validator`, please replace it by `field_validator` manually. + # Check https://docs.pydantic.dev/dev-v2/migration/#changes-to-validators for more information. @validator("metadata_submission", pre=True, always=True) def populate_roles(cls, metadata_submission, values): owners = set(values.get("owners", [])) diff --git a/pyproject.toml b/pyproject.toml index d41e497b..9ebf255e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -8,37 +8,163 @@ readme = "README.md" requires-python = ">=3.9" dynamic = ["version"] dependencies = [ - # pinned because recent versions throw an error when upgrading through - # the api at nmdc_server/jobs.py:34 - "alembic==1.5.8", - "authlib==1.3.1", - "celery[redis]", - "click", - "cryptography", - # https://github.com/microbiomedata/nmdc-server/actions/runs/5671884086/job/15369921721 - "dnspython==2.6.1", - "fastapi==0.71.0", - "factory-boy==3.2.1", - "httpx==0.23.0", - "ipython==8.10.0", - "itsdangerous==2.0.1", - "mypy<0.920", + "alembic==1.13.3", + "amqp==5.2.0", + "annotated-types==0.7.0", + "antlr4-python3-runtime==4.9.3", + "anyio==4.6.0", + "appdirs==1.4.4", + "arrow==1.3.0", + "asttokens==2.4.1", + "async-timeout==4.0.3", + "attrs==24.2.0", + "authlib==1.3.2", + "babel==2.16.0", + "beautifulsoup4==4.12.3", + "billiard==4.2.1", + "celery==5.4.0", + "certifi==2024.8.30", + "cffi==1.17.1", + "cfgraph==0.2.1", + "chardet==5.2.0", + "charset-normalizer==3.4.0", + "click==8.1.7", + "click-didyoumean==0.3.1", + "click-plugins==1.1.1", + "click-repl==0.3.0", + "colorama==0.4.6", + "cryptography==43.0.1", + "curies==0.8.0", + "decorator==5.1.1", + "deprecated==1.2.14", + "dnspython==2.7.0", + "editorconfig==0.12.4", + "et-xmlfile==1.1.0", + "exceptiongroup==1.2.2", + "executing==2.1.0", + "factory-boy==3.3.1", + "faker==30.3.0", + "fastapi==0.115.0", + "flexcache==0.3", + "flexparser==0.3.1", + "fqdn==1.5.1", + "ghp-import==2.1.0", + "graphviz==0.20.3", + "greenlet==3.1.1", + "h11==0.14.0", + "hbreader==0.9.1", + "httpcore==1.0.6", + "httpx==0.27.2", + "idna==3.10", + "importlib-metadata==4.12.0", + "iniconfig==2.0.0", + "ipython==8.18.1", + "isodate==0.7.2", + "isoduration==20.11.0", + "itsdangerous==2.2.0", + "jedi==0.19.1", + "jinja2==3.1.4", + "jsbeautifier==1.15.1", + "json-flattener==0.1.9", + "jsonasobj==1.3.1", + "jsonasobj2==1.0.4", + "jsonpatch==1.33", + "jsonpath-ng==1.6.1", + "jsonpointer==3.0.0", + "jsonschema==4.23.0", + "jsonschema-specifications==2024.10.1", + "kombu==5.4.2", + "linkml==1.8.4", + "linkml-dataops==0.1.0", + "linkml-runtime==1.8.3", + "mako==1.3.5", + "markdown==3.7", + "markupsafe==3.0.1", + "matplotlib-inline==0.1.7", + "mergedeep==1.3.4", + "mkdocs==1.6.1", + "mkdocs-get-deps==0.2.0", + "mkdocs-material==9.5.40", + "mkdocs-material-extensions==1.3.1", + "mkdocs-mermaid2-plugin==0.6.0", + "mkdocs-redirects==1.2.1", + "mypy==1.11.2", + "mypy-extensions==1.0.0", "nmdc-schema==11.0.1", "nmdc-submission-schema==11.0.0", "nmdc-geoloc-tools==0.1.1", - "pint==0.18", - "psycopg2==2.9.3", - "pydantic==1.10.13", - "pymongo>=4.0.0", - "python-dateutil", - "python-dotenv", - "requests==2.32.2", - "sentry-sdk[celery,sqlalchemy]", - "sqlalchemy~=1.4", - "starlette==0.17.1", - "typing-extensions==4.2.0", - # pinned 3rd party dependencies - "importlib-metadata==4.12.0", + "openpyxl==3.1.5", + "packaging==24.1", + "paginate==0.5.7", + "parse==1.20.2", + "parso==0.8.4", + "pathspec==0.12.1", + "pexpect==4.9.0", + "pint==0.24.3", + "platformdirs==4.3.6", + "pluggy==1.5.0", + "ply==3.11", + "prefixcommons==0.1.12", + "prefixmaps==0.2.5", + "prompt-toolkit==3.0.48", + "psycopg2==2.9.9", + "ptyprocess==0.7.0", + "pure-eval==0.2.3", + "pycparser==2.22", + "pydantic==2.9.2", + "pydantic-settings==2.4.0", + "pydantic-core==2.23.4", + "pygments==2.18.0", + "pyjsg==0.11.10", + "pymdown-extensions==10.11.2", + "pymongo==4.10.1", + "pyparsing==3.1.4", + "pyshex==0.8.1", + "pyshexc==0.9.1", + "pytest==8.3.3", + "pytest-logging==2015.11.4", + "python-dateutil==2.9.0.post0", + "python-dotenv==1.0.1", + "pytrie==0.4.0", + "pyyaml==6.0.2", + "pyyaml-env-tag==0.1", + "rdflib==6.2.0", + "rdflib-jsonld==0.6.1", + "rdflib-shim==1.0.3", + "redis==5.1.1", + "referencing==0.35.1", + "regex==2024.9.11", + "requests==2.32.3", + "rfc3339-validator==0.1.4", + "rfc3987==1.3.8", + "rpds-py==0.20.0", + "ruamel-yaml==0.18.6", + "ruamel-yaml-clib==0.2.8", + "sentry-sdk==2.16.0", + "setuptools==75.1.0", + "shexjsg==0.8.2", + "six==1.16.0", + "sniffio==1.3.1", + "sortedcontainers==2.4.0", + "soupsieve==2.6", + "sparqlslurper==0.5.1", + "sparqlwrapper==2.0.0", + "sqlalchemy<2", + "stack-data==0.6.3", + "starlette==0.38.6", + "tomli==2.0.2", + "traitlets==5.14.3", + "types-python-dateutil==2.9.0.20241003", + "typing-extensions==4.12.2", + "tzdata==2024.2", + "uri-template==1.3.0", + "urllib3==2.2.3", + "vine==5.1.0", + "watchdog==5.0.3", + "wcwidth==0.2.13", + "webcolors==24.8.0", + "wrapt==1.16.0", + "zipp==3.20.2", ] [project.scripts] diff --git a/tests/test_auth.py b/tests/test_auth.py index 7d346417..0ea22ef4 100644 --- a/tests/test_auth.py +++ b/tests/test_auth.py @@ -10,11 +10,11 @@ def test_login(client: TestClient): resp = client.request( method="get", url=f"/auth/login?redirect_uri={allowed_redirect_uri}/whatever", - allow_redirects=False, + follow_redirects=False, ) assert resp.status_code == 302 - assert resp.next.url.startswith(f"{settings.orcid_base_url}/oauth/authorize") # type: ignore + assert str(resp.next_request.url).startswith(f"{settings.orcid_base_url}/oauth/authorize") # type: ignore def test_current_user(client: TestClient, logged_in_user): diff --git a/tests/test_download.py b/tests/test_download.py index 0f462a1e..225cdf7c 100644 --- a/tests/test_download.py +++ b/tests/test_download.py @@ -32,20 +32,24 @@ def test_bulk_download_query(db: Session): qs = query.DataObjectQuerySchema() rows = qs.execute(db).all() assert len(rows) == 0 - assert qs.aggregate(db) == { - "size": 0, - "count": 0, - } + + data_object_agg_obj = qs.aggregate(db) + assert data_object_agg_obj.size == 0 + assert data_object_agg_obj.count == 0 qs = query.DataObjectQuerySchema(data_object_filter=[{"workflow": "nmdc:RawData"}]) rows = qs.execute(db).all() assert [raw1.id] == [d.id for d in rows] - assert qs.aggregate(db) == {"size": raw1.file_size_bytes, "count": 1} + data_object_agg_obj = qs.aggregate(db) + assert data_object_agg_obj.size == raw1.file_size_bytes + assert data_object_agg_obj.count == 1 qs = query.DataObjectQuerySchema(data_object_filter=[{"file_type": "ftype1"}]) rows = qs.execute(db).all() assert [raw1.id] == [d.id for d in rows] - assert qs.aggregate(db) == {"size": raw1.file_size_bytes, "count": 1} + data_object_agg_obj = qs.aggregate(db) + assert data_object_agg_obj.size == raw1.file_size_bytes + assert data_object_agg_obj.count == 1 def test_generate_bulk_download(db: Session, client: TestClient, logged_in_user): diff --git a/tests/test_submission.py b/tests/test_submission.py index 35d4ea4a..82988359 100644 --- a/tests/test_submission.py +++ b/tests/test_submission.py @@ -15,7 +15,7 @@ def suggest_payload(): return [ {"row": 1, "data": {"foo": "bar", "lat_lon": "44.058648, -123.095277"}}, - {"row": 3, "data": {"elev": 0, "lat_lon": "44.046389 -123.051910"}}, + {"row": 3, "data": {"elev": "0", "lat_lon": "44.046389 -123.051910"}}, {"row": 4, "data": {"foo": "bar"}}, {"row": 5, "data": {"lat_lon": "garbage foo bar"}}, ] diff --git a/tox.ini b/tox.ini index a72c6c05..edb0ff75 100644 --- a/tox.ini +++ b/tox.ini @@ -14,6 +14,7 @@ deps = requests setenv = NMDC_ENVIRONMENT = testing + SQLALCHEMY_SILENCE_UBER_WARNING=1 passenv = NMDC_TESTING_DATABASE_URI commands = pytest {posargs} tests/ @@ -60,6 +61,7 @@ commands = black nmdc_server tests [pytest] addopts = --verbose --showlocals +filterwarnings = ignore::pydantic.PydanticDeprecatedSince20 [flake8] format = pylint @@ -74,4 +76,4 @@ extend-ignore = N805 # conflicts with pydantic validators N815 # conflicts with nmdc column names N818 # conflicts with existing code (Error suffix) - E711 # conflicts with sqlalchemy comparisons \ No newline at end of file + E711 # conflicts with sqlalchemy comparisons From f35e16307a50b5f84c437ff4faea5359145580ed Mon Sep 17 00:00:00 2001 From: naglepuff Date: Thu, 10 Oct 2024 11:44:28 -0400 Subject: [PATCH 02/14] WIP toolbar --- nmdc_server/app.py | 3 +++ pyproject.toml | 1 + 2 files changed, 4 insertions(+) diff --git a/nmdc_server/app.py b/nmdc_server/app.py index 529e2df5..69facce6 100644 --- a/nmdc_server/app.py +++ b/nmdc_server/app.py @@ -1,6 +1,7 @@ import typing import sentry_sdk +from debug_toolbar.middleware import DebugToolbarMiddleware from fastapi import FastAPI from fastapi.middleware.cors import CORSMiddleware from fastapi.responses import RedirectResponse @@ -35,7 +36,9 @@ def create_app(env: typing.Mapping[str, str]) -> FastAPI: version=__version__, docs_url="/api/docs", openapi_url="/api/openapi.json", + debug=True, ) + app.add_middleware(DebugToolbarMiddleware) @app.get("/docs", response_class=RedirectResponse, status_code=301, include_in_schema=False) async def redirect_docs(): diff --git a/pyproject.toml b/pyproject.toml index 9ebf255e..c76783ba 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -165,6 +165,7 @@ dependencies = [ "webcolors==24.8.0", "wrapt==1.16.0", "zipp==3.20.2", + "fastapi-debug-toolbar", ] [project.scripts] From dd088b196f24607117755156c1589b70bd282e18 Mon Sep 17 00:00:00 2001 From: naglepuff Date: Thu, 10 Oct 2024 12:57:19 -0400 Subject: [PATCH 03/14] Create index on download/data object association This should speed up biosample search, since it returns download statistics for each data object for each biosample. --- .../2ec2d0b4f840_create_data_object_id_idx.py | 35 +++++++++++++++++++ nmdc_server/models.py | 4 +++ 2 files changed, 39 insertions(+) create mode 100644 nmdc_server/migrations/versions/2ec2d0b4f840_create_data_object_id_idx.py diff --git a/nmdc_server/migrations/versions/2ec2d0b4f840_create_data_object_id_idx.py b/nmdc_server/migrations/versions/2ec2d0b4f840_create_data_object_id_idx.py new file mode 100644 index 00000000..87024280 --- /dev/null +++ b/nmdc_server/migrations/versions/2ec2d0b4f840_create_data_object_id_idx.py @@ -0,0 +1,35 @@ +"""Create index on column `data_object_id` for table `bulk_download_data_object` + +Revision ID: 2ec2d0b4f840 +Revises: 5fb9910ca8e6 +Create Date: 2024-10-10 16:48:37.051479 + +""" + +from typing import Optional + +import sqlalchemy as sa +from alembic import op + +# revision identifiers, used by Alembic. +revision: str = "2ec2d0b4f840" +down_revision: Optional[str] = "5fb9910ca8e6" +branch_labels: Optional[str] = None +depends_on: Optional[str] = None + + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.create_index( + "bulk_download_data_object_id_idx", + "bulk_download_data_object", + ["data_object_id"], + unique=False, + ) + # ### end Alembic commands ### + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.drop_index("bulk_download_data_object_id_idx", table_name="bulk_download_data_object") + # ### end Alembic commands ### diff --git a/nmdc_server/models.py b/nmdc_server/models.py index 88c94f3b..b4f1b4b4 100644 --- a/nmdc_server/models.py +++ b/nmdc_server/models.py @@ -12,6 +12,7 @@ Enum, Float, ForeignKey, + Index, Integer, LargeBinary, String, @@ -834,6 +835,9 @@ class BulkDownloadDataObject(Base): ) +Index("bulk_download_data_object_id_idx", BulkDownloadDataObject.data_object_id) + + class EnvoTree(Base): __tablename__ = "envo_tree" From 79186572a43c7a3b327a3f3076084ffa9d57503c Mon Sep 17 00:00:00 2001 From: naglepuff Date: Thu, 10 Oct 2024 13:39:29 -0400 Subject: [PATCH 04/14] Remove unused import --- .../versions/2ec2d0b4f840_create_data_object_id_idx.py | 1 - 1 file changed, 1 deletion(-) diff --git a/nmdc_server/migrations/versions/2ec2d0b4f840_create_data_object_id_idx.py b/nmdc_server/migrations/versions/2ec2d0b4f840_create_data_object_id_idx.py index 87024280..ed6fb698 100644 --- a/nmdc_server/migrations/versions/2ec2d0b4f840_create_data_object_id_idx.py +++ b/nmdc_server/migrations/versions/2ec2d0b4f840_create_data_object_id_idx.py @@ -8,7 +8,6 @@ from typing import Optional -import sqlalchemy as sa from alembic import op # revision identifiers, used by Alembic. From 792a7fb440f762739c6ed18afb72ad9982b63aca Mon Sep 17 00:00:00 2001 From: naglepuff Date: Thu, 10 Oct 2024 17:37:21 -0400 Subject: [PATCH 05/14] Fix mypy errors --- nmdc_server/api.py | 4 ++-- nmdc_server/attribute_units.py | 7 ++++--- nmdc_server/config.py | 3 +++ nmdc_server/crud.py | 2 +- nmdc_server/ingest/biosample.py | 6 +++--- nmdc_server/ingest/common.py | 4 ++-- nmdc_server/ingest/data_object.py | 2 +- nmdc_server/ingest/doi.py | 2 +- nmdc_server/ingest/omics_processing.py | 6 +++--- nmdc_server/ingest/study.py | 5 ++--- nmdc_server/migrations/env.py | 2 +- nmdc_server/query.py | 26 +++++++++++--------------- nmdc_server/utils.py | 3 ++- tests/test_app.py | 2 +- 14 files changed, 37 insertions(+), 37 deletions(-) diff --git a/nmdc_server/api.py b/nmdc_server/api.py index 131ced5e..82c8dcf8 100644 --- a/nmdc_server/api.py +++ b/nmdc_server/api.py @@ -47,8 +47,8 @@ async def get_version() -> schemas.VersionInfo: # get the current user information @router.get("/me", tags=["user"], name="Return the current user name") -async def me(user: User = Depends(get_current_user)) -> Optional[schemas.User]: - return schemas.User(**user) +async def me(user: User = Depends(get_current_user)): + return user # autocomplete search diff --git a/nmdc_server/attribute_units.py b/nmdc_server/attribute_units.py index e26f6ee1..ff3e2926 100644 --- a/nmdc_server/attribute_units.py +++ b/nmdc_server/attribute_units.py @@ -1,6 +1,7 @@ from typing import Dict, Optional -from pint import Quantity, Unit, UnitRegistry +from pint import Quantity, UnitRegistry +from pint.facets.plain.unit import PlainUnit _registry = UnitRegistry() @@ -8,14 +9,14 @@ # hard code relevant attributes here. -_unit_info: Dict[str, Dict[str, Unit]] = { +_unit_info: Dict[str, Dict[str, PlainUnit]] = { "biosample": { "depth": _registry("meter").units, } } -def get_attribute_units(table: str, attribute: str) -> Optional[Unit]: +def get_attribute_units(table: str, attribute: str) -> Optional[PlainUnit]: return _unit_info.get(table, {}).get(attribute) diff --git a/nmdc_server/config.py b/nmdc_server/config.py index db19af59..5102648d 100644 --- a/nmdc_server/config.py +++ b/nmdc_server/config.py @@ -1,5 +1,8 @@ +from __future__ import annotations + import os from typing import Optional + from pydantic_settings import BaseSettings, SettingsConfigDict diff --git a/nmdc_server/crud.py b/nmdc_server/crud.py index 273e5da6..e0a62562 100644 --- a/nmdc_server/crud.py +++ b/nmdc_server/crud.py @@ -27,7 +27,7 @@ def get_or_create( else: params = dict(**kwargs) params.update(defaults or {}) - instance = model(**params) # type: ignore + instance = model(**params) db.add(instance) return instance, True diff --git a/nmdc_server/ingest/biosample.py b/nmdc_server/ingest/biosample.py index f74f69d3..da4ccf26 100644 --- a/nmdc_server/ingest/biosample.py +++ b/nmdc_server/ingest/biosample.py @@ -3,7 +3,8 @@ from datetime import datetime from typing import Any, Dict -from pydantic import field_validator, model_validator, validator +from pydantic import field_validator +from pydantic.v1 import root_validator, validator from pymongo.cursor import Cursor from sqlalchemy.orm import Session @@ -20,8 +21,7 @@ class Biosample(BiosampleCreate): _extract_value = validator("*", pre=True, allow_reuse=True)(extract_value) - @model_validator(mode="before") - @classmethod + @root_validator(pre=True) def extract_extras(cls, values): if "lat_lon" in values: if "latitude" in values["lat_lon"] and "longitude" in values["lat_lon"]: diff --git a/nmdc_server/ingest/common.py b/nmdc_server/ingest/common.py index 58717f17..0820ada8 100644 --- a/nmdc_server/ingest/common.py +++ b/nmdc_server/ingest/common.py @@ -1,6 +1,6 @@ import logging from datetime import datetime -from typing import Any, Dict, Set, Union +from typing import Any, Dict, Optional, Set, Union from pydantic import BaseModel from sqlalchemy.exc import IntegrityError @@ -45,7 +45,7 @@ def extract_value(value: Any) -> Any: def extract_extras( - cls: BaseModel, values: Dict[str, Any], exclude: Set[str] = None + cls: BaseModel, values: Dict[str, Any], exclude: Optional[Set[str]] = None ) -> Dict[str, Any]: # Move unknown attributes into values['annotations'] fields = set(cls.__fields__.keys()) diff --git a/nmdc_server/ingest/data_object.py b/nmdc_server/ingest/data_object.py index 14573302..5569193d 100644 --- a/nmdc_server/ingest/data_object.py +++ b/nmdc_server/ingest/data_object.py @@ -14,7 +14,7 @@ def load(db: Session, cursor: Cursor, file_types: List[Dict[str, Any]]): logger = get_logger(__name__) - fields = set(DataObjectCreate.__fields__.keys()) | {"data_object_type"} + fields = set(DataObjectCreate.model_fields.keys()) | {"data_object_type"} file_type_map: Dict[str, Tuple[str, str]] = {} # Load descriptors from mongo collection. diff --git a/nmdc_server/ingest/doi.py b/nmdc_server/ingest/doi.py index 19cd881d..7131b12b 100644 --- a/nmdc_server/ingest/doi.py +++ b/nmdc_server/ingest/doi.py @@ -3,9 +3,9 @@ import requests from requests.adapters import HTTPAdapter from requests.models import Response -from requests.packages.urllib3.util.retry import Retry from sqlalchemy.dialects.postgresql import insert from sqlalchemy.orm import Session +from urllib3.util.retry import Retry from nmdc_server.logger import get_logger from nmdc_server.models import DOIInfo, DOIType diff --git a/nmdc_server/ingest/omics_processing.py b/nmdc_server/ingest/omics_processing.py index e8bd510a..c4dbbeb8 100644 --- a/nmdc_server/ingest/omics_processing.py +++ b/nmdc_server/ingest/omics_processing.py @@ -3,7 +3,8 @@ from datetime import datetime from typing import Any, Dict, Optional -from pydantic import field_validator, model_validator, validator +from pydantic import field_validator +from pydantic.v1 import root_validator, validator from pymongo.collection import Collection from pymongo.cursor import Cursor from pymongo.database import Database @@ -35,8 +36,7 @@ class OmicsProcessing(OmicsProcessingCreate): _extract_value = validator("*", pre=True, allow_reuse=True)(extract_value) - @model_validator(mode="before") - @classmethod + @root_validator(pre=True) def extract_extras(cls, values): return extract_extras(cls, values) diff --git a/nmdc_server/ingest/study.py b/nmdc_server/ingest/study.py index 3c01a5d4..be2eed7d 100644 --- a/nmdc_server/ingest/study.py +++ b/nmdc_server/ingest/study.py @@ -2,7 +2,7 @@ from typing import Optional import requests -from pydantic import model_validator, validator +from pydantic.v1 import root_validator, validator from pymongo.cursor import Cursor from sqlalchemy.orm import Session @@ -41,8 +41,7 @@ def get_or_create_pi(db: Session, name: str, url: Optional[str], orcid: Optional class Study(StudyCreate): _extract_value = validator("*", pre=True, allow_reuse=True)(extract_value) - @model_validator(mode="before") - @classmethod + @root_validator(pre=True) def extract_extras(cls, values): return extract_extras(cls, values) diff --git a/nmdc_server/migrations/env.py b/nmdc_server/migrations/env.py index 7ece1b83..e7dac256 100644 --- a/nmdc_server/migrations/env.py +++ b/nmdc_server/migrations/env.py @@ -13,7 +13,7 @@ # Interpret the config file for Python logging. # This line sets up loggers basically. if config.attributes.get("configure_logger", True): - fileConfig(config.config_file_name) + fileConfig(config.config_file_name) # type: ignore # add your model's MetaData object here # for 'autogenerate' support diff --git a/nmdc_server/query.py b/nmdc_server/query.py index 6de7063f..e4d80540 100644 --- a/nmdc_server/query.py +++ b/nmdc_server/query.py @@ -9,7 +9,7 @@ from itertools import groupby from typing import Any, Dict, Iterator, List, Optional, Tuple, Union -from pydantic import ConfigDict, BaseModel, Field, PositiveInt +from pydantic import BaseModel, ConfigDict, Field, PositiveInt from sqlalchemy import ARRAY, Column, and_, cast, desc, func, inspect, or_ from sqlalchemy.orm import Query, Session, aliased, with_expression from sqlalchemy.orm.util import AliasedClass @@ -184,7 +184,7 @@ def compare(self) -> ClauseElement: elif self.op == Operation.like: return column.ilike(f"%{self.value}%") if hasattr(model, "annotations"): - json_field = model.annotations # type: ignore + json_field = model.annotations else: raise InvalidAttributeException(self.table.value, self.field) if self.op == Operation.like: @@ -208,12 +208,8 @@ def compare(self) -> ClauseElement: return and_(column >= self.value[0], column <= self.value[1]) if hasattr(model, "annotations"): return and_( - func.nmdc_compare( - model.annotations[self.field].astext, ">=", self.value[0] # type: ignore - ), - func.nmdc_compare( - model.annotations[self.field].astext, "<=", self.value[1] # type: ignore - ), + func.nmdc_compare(model.annotations[self.field].astext, ">=", self.value[0]), + func.nmdc_compare(model.annotations[self.field].astext, "<=", self.value[1]), ) else: raise InvalidAttributeException(self.table.value, self.field) @@ -393,8 +389,8 @@ def get_query_range( db: Session, column: Column, subquery: Any, - minimum: NumericValue = None, - maximum: NumericValue = None, + minimum: Optional[NumericValue] = None, + maximum: Optional[NumericValue] = None, ) -> Tuple[Optional[NumericValue], Optional[NumericValue]]: """Get the range of a numeric/datetime quantity matching the conditions. @@ -416,9 +412,9 @@ def get_query_range( def validate_binning_args( self, attribute: str, - minimum: NumericValue = None, - maximum: NumericValue = None, - resolution: DateBinResolution = None, + minimum: Optional[NumericValue] = None, + maximum: Optional[NumericValue] = None, + resolution: Optional[DateBinResolution] = None, ): """Raise an exception if binning arguments aren't valid for the data type.""" # TODO: Validation like this should happen at the schema layer, but it requires refactoring @@ -454,8 +450,8 @@ def binned_facet( self, db: Session, attribute: str, - minimum: NumericValue = None, - maximum: NumericValue = None, + minimum: Optional[NumericValue] = None, + maximum: Optional[NumericValue] = None, **kwargs, ) -> Tuple[List[NumericValue], List[int]]: """Perform a binned faceting aggregation on an attribute.""" diff --git a/nmdc_server/utils.py b/nmdc_server/utils.py index 5b97e5f9..58df3b84 100644 --- a/nmdc_server/utils.py +++ b/nmdc_server/utils.py @@ -28,8 +28,9 @@ def log_extras(req: Request) -> Dict[str, Any]: :param req: the current request. """ + ip = req.client.host if req.client else "" return { - "ip": req.client.host, + "ip": ip, "method": req.method, "url": req.url, } diff --git a/tests/test_app.py b/tests/test_app.py index 69d886d0..d251535c 100644 --- a/tests/test_app.py +++ b/tests/test_app.py @@ -4,7 +4,7 @@ import pytest from fastapi.testclient import TestClient -from requests.models import Response +from httpx import Response from sqlalchemy.orm.session import Session import nmdc_server From e1ff4bacee3c3ac971c821857e48896a279b479b Mon Sep 17 00:00:00 2001 From: naglepuff Date: Thu, 10 Oct 2024 17:43:34 -0400 Subject: [PATCH 06/14] Format --- nmdc_server/bulk_download_schema.py | 3 ++- nmdc_server/schemas.py | 12 ++++++++---- nmdc_server/schemas_submission.py | 5 +++-- tests/test_auth.py | 3 ++- 4 files changed, 15 insertions(+), 8 deletions(-) diff --git a/nmdc_server/bulk_download_schema.py b/nmdc_server/bulk_download_schema.py index f1870c76..2d8aea37 100644 --- a/nmdc_server/bulk_download_schema.py +++ b/nmdc_server/bulk_download_schema.py @@ -2,10 +2,11 @@ from typing import List from uuid import UUID +from pydantic import ConfigDict + from nmdc_server.data_object_filters import DataObjectFilter from nmdc_server.query import ConditionSchema from nmdc_server.schemas import FileDownloadMetadata -from pydantic import ConfigDict # schemas related to bulk download endpoints extracted diff --git a/nmdc_server/schemas.py b/nmdc_server/schemas.py index 40b5fcea..dfabb0c3 100644 --- a/nmdc_server/schemas.py +++ b/nmdc_server/schemas.py @@ -237,7 +237,8 @@ class StudyBase(AnnotatedBase): study_category: Optional[str] = None children: Optional[List[Study]] = [] - # TODO[pydantic]: We couldn't refactor the `validator`, please replace it by `field_validator` manually. + # TODO[pydantic]: We couldn't refactor the `validator`, please replace it by `field_validator` + # manually. # Check https://docs.pydantic.dev/dev-v2/migration/#changes-to-validators for more information. @validator("principal_investigator_websites", pre=True, each_item=True) def replace_websites(cls, study_website: Union[models.StudyWebsite, str]) -> str: @@ -255,7 +256,8 @@ class OmicsCounts(BaseModel): type: str count: int - # TODO[pydantic]: We couldn't refactor the `validator`, please replace it by `field_validator` manually. + # TODO[pydantic]: We couldn't refactor the `validator`, please replace it by `field_validator` + # manually. # Check https://docs.pydantic.dev/dev-v2/migration/#changes-to-validators for more information. @validator("count", pre=True, always=True) def insert_zero(cls, v): @@ -336,7 +338,8 @@ class OmicsProcessing(OmicsProcessingBase): omics_data: List["OmicsTypes"] outputs: List["DataObject"] - # TODO[pydantic]: We couldn't refactor the `validator`, please replace it by `field_validator` manually. + # TODO[pydantic]: We couldn't refactor the `validator`, please replace it by `field_validator` + # manually. # Check https://docs.pydantic.dev/dev-v2/migration/#changes-to-validators for more information. @validator("biosample_ids") @classmethod @@ -372,7 +375,8 @@ class DataObject(DataObjectBase): selected: Optional[bool] = None model_config = ConfigDict(from_attributes=True) - # TODO[pydantic]: We couldn't refactor the `validator`, please replace it by `field_validator` manually. + # TODO[pydantic]: We couldn't refactor the `validator`, please replace it by `field_validator` + # manually. # Check https://docs.pydantic.dev/dev-v2/migration/#changes-to-validators for more information. @validator("url") def replace_url(cls, url, values): diff --git a/nmdc_server/schemas_submission.py b/nmdc_server/schemas_submission.py index 6a17a162..d80144cc 100644 --- a/nmdc_server/schemas_submission.py +++ b/nmdc_server/schemas_submission.py @@ -3,7 +3,7 @@ from typing import Any, Dict, List, Optional from uuid import UUID -from pydantic import ConfigDict, BaseModel, validator +from pydantic import BaseModel, ConfigDict, validator from nmdc_server import schemas from nmdc_server.models import SubmissionEditorRole @@ -121,7 +121,8 @@ class SubmissionMetadataSchema(SubmissionMetadataSchemaCreate): permission_level: Optional[str] = None model_config = ConfigDict(from_attributes=True) - # TODO[pydantic]: We couldn't refactor the `validator`, please replace it by `field_validator` manually. + # TODO[pydantic]: We couldn't refactor the `validator`, please replace it by `field_validator` + # manually. # Check https://docs.pydantic.dev/dev-v2/migration/#changes-to-validators for more information. @validator("metadata_submission", pre=True, always=True) def populate_roles(cls, metadata_submission, values): diff --git a/tests/test_auth.py b/tests/test_auth.py index 0ea22ef4..2c8c5d86 100644 --- a/tests/test_auth.py +++ b/tests/test_auth.py @@ -14,7 +14,8 @@ def test_login(client: TestClient): ) assert resp.status_code == 302 - assert str(resp.next_request.url).startswith(f"{settings.orcid_base_url}/oauth/authorize") # type: ignore + assert resp.next_request + assert str(resp.next_request.url).startswith(f"{settings.orcid_base_url}/oauth/authorize") def test_current_user(client: TestClient, logged_in_user): From 15defb3a22623171621c3fc3446213b9404663b5 Mon Sep 17 00:00:00 2001 From: naglepuff Date: Tue, 15 Oct 2024 14:09:19 -0400 Subject: [PATCH 07/14] Use new paradigm for fastapi lifecycle events --- nmdc_server/app.py | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/nmdc_server/app.py b/nmdc_server/app.py index 69facce6..3acd37f9 100644 --- a/nmdc_server/app.py +++ b/nmdc_server/app.py @@ -1,4 +1,5 @@ import typing +from contextlib import asynccontextmanager import sentry_sdk from debug_toolbar.middleware import DebugToolbarMiddleware @@ -25,6 +26,16 @@ def attach_sentry(app: FastAPI): def create_app(env: typing.Mapping[str, str]) -> FastAPI: + def generate_and_mount_static_files(): + static_path = initialize_static_directory(remove_existing=True) + generate_submission_schema_files(directory=static_path) + app.mount("/static", StaticFiles(directory=static_path), name="static") + + @asynccontextmanager + async def lifespan(app: FastAPI): + generate_and_mount_static_files() + yield + app = FastAPI( title="NMDC Data and Submission Portal API", description=""" @@ -37,6 +48,7 @@ def create_app(env: typing.Mapping[str, str]) -> FastAPI: docs_url="/api/docs", openapi_url="/api/openapi.json", debug=True, + lifespan=lifespan, ) app.add_middleware(DebugToolbarMiddleware) @@ -44,12 +56,6 @@ def create_app(env: typing.Mapping[str, str]) -> FastAPI: async def redirect_docs(): return "/api/docs" - @app.on_event("startup") - async def generate_and_mount_static_files(): - static_path = initialize_static_directory(remove_existing=True) - generate_submission_schema_files(directory=static_path) - app.mount("/static", StaticFiles(directory=static_path), name="static") - attach_sentry(app) errors.attach_error_handlers(app) app.include_router(api.router, prefix="/api") From f4afb9838407f676eaa824233ac75c3c33f96de8 Mon Sep 17 00:00:00 2001 From: naglepuff Date: Tue, 15 Oct 2024 14:21:37 -0400 Subject: [PATCH 08/14] Enable tracing through sentry --- nmdc_server/app.py | 2 ++ nmdc_server/config.py | 6 ++++++ 2 files changed, 8 insertions(+) diff --git a/nmdc_server/app.py b/nmdc_server/app.py index 3acd37f9..784c66e6 100644 --- a/nmdc_server/app.py +++ b/nmdc_server/app.py @@ -22,6 +22,8 @@ def attach_sentry(app: FastAPI): sentry_sdk.init( dsn=settings.sentry_dsn, integrations=[SqlalchemyIntegration()], + enable_tracing=settings.sentry_tracing_enabled, + traces_sample_rate=settings.sentry_traces_sample_rate, ) diff --git a/nmdc_server/config.py b/nmdc_server/config.py index 5102648d..c36fa912 100644 --- a/nmdc_server/config.py +++ b/nmdc_server/config.py @@ -69,6 +69,12 @@ def orcid_openid_config_url(self) -> str: sentry_dsn: Optional[str] = None + # Enable/disable and configure tracing through environment + # variables to lessen friction when fine-tuning settings + # for useful tracing. + sentry_tracing_enabled: bool = False + sentry_traces_sample_rate: float = 0.0 + print_sql: bool = False # App settings related to UI behavior From 204eed2a28fbab7235347ca87d410065e503f6ad Mon Sep 17 00:00:00 2001 From: naglepuff Date: Wed, 16 Oct 2024 13:49:44 -0400 Subject: [PATCH 09/14] Use pydantic v2 `field_validator` --- nmdc_server/schemas.py | 57 +++++++++++++------------------ nmdc_server/schemas_submission.py | 19 +++++------ 2 files changed, 32 insertions(+), 44 deletions(-) diff --git a/nmdc_server/schemas.py b/nmdc_server/schemas.py index dfabb0c3..045d59cb 100644 --- a/nmdc_server/schemas.py +++ b/nmdc_server/schemas.py @@ -10,12 +10,12 @@ from datetime import date, datetime from enum import Enum from importlib.metadata import version -from typing import Any, Dict, List, Optional, Union +from typing import Annotated, Any, Dict, List, Optional, Union from urllib.parse import quote from uuid import UUID from pint import Unit -from pydantic import BaseModel, ConfigDict, Field, validator +from pydantic import BaseModel, BeforeValidator, ConfigDict, Field, ValidationInfo, field_validator from sqlalchemy import BigInteger, Column, DateTime, Float, Integer, LargeBinary, String from sqlalchemy.dialects.postgresql.json import JSONB @@ -221,8 +221,17 @@ class DOIInfo(BaseModel): model_config = ConfigDict(from_attributes=True) +def replace_website(site: Union[str, models.StudyWebsite]) -> str: + if isinstance(site, models.StudyWebsite): + return site.website.url + return site + + +PiSite = Annotated[str, BeforeValidator(replace_website)] + + class StudyBase(AnnotatedBase): - principal_investigator_websites: Optional[List[str]] = [] + principal_investigator_websites: Optional[List[PiSite]] = [] gold_name: str = "" gold_description: str = "" scientific_objective: str = "" @@ -237,15 +246,6 @@ class StudyBase(AnnotatedBase): study_category: Optional[str] = None children: Optional[List[Study]] = [] - # TODO[pydantic]: We couldn't refactor the `validator`, please replace it by `field_validator` - # manually. - # Check https://docs.pydantic.dev/dev-v2/migration/#changes-to-validators for more information. - @validator("principal_investigator_websites", pre=True, each_item=True) - def replace_websites(cls, study_website: Union[models.StudyWebsite, str]) -> str: - if isinstance(study_website, str): - return study_website - return study_website.website.url - class StudyCreate(StudyBase): principal_investigator_id: Optional[UUID] = None @@ -256,10 +256,7 @@ class OmicsCounts(BaseModel): type: str count: int - # TODO[pydantic]: We couldn't refactor the `validator`, please replace it by `field_validator` - # manually. - # Check https://docs.pydantic.dev/dev-v2/migration/#changes-to-validators for more information. - @validator("count", pre=True, always=True) + @field_validator("count", mode="before") def insert_zero(cls, v): return v or 0 @@ -338,16 +335,13 @@ class OmicsProcessing(OmicsProcessingBase): omics_data: List["OmicsTypes"] outputs: List["DataObject"] - # TODO[pydantic]: We couldn't refactor the `validator`, please replace it by `field_validator` - # manually. - # Check https://docs.pydantic.dev/dev-v2/migration/#changes-to-validators for more information. - @validator("biosample_ids") + @field_validator("biosample_ids") @classmethod - def set_biosample_ids(cls, biosample_ids: list[str], values: dict[str, Any]) -> list[str]: + def set_biosample_ids(cls, biosample_ids: list[str], info: ValidationInfo) -> list[str]: # Only capture biosample IDs in responses - biosample_objects: list[BiosampleBase] = values.get("biosample_inputs", []) + biosample_objects: list[BiosampleBase] = info.data.get("biosample_inputs", []) biosample_ids = biosample_ids + [biosample.id for biosample in biosample_objects] - values.pop("biosample_inputs") + info.data.pop("biosample_inputs") return biosample_ids @@ -375,12 +369,9 @@ class DataObject(DataObjectBase): selected: Optional[bool] = None model_config = ConfigDict(from_attributes=True) - # TODO[pydantic]: We couldn't refactor the `validator`, please replace it by `field_validator` - # manually. - # Check https://docs.pydantic.dev/dev-v2/migration/#changes-to-validators for more information. - @validator("url") - def replace_url(cls, url, values): - id_str = quote(values["id"]) + @field_validator("url") + def replace_url(cls, url, info: ValidationInfo): + id_str = quote(info.data["id"]) return f"/api/data_object/{id_str}/download" if url else None # Determine if the data object is selected by the provided filter. @@ -607,9 +598,9 @@ class MetabolomicsAnalysis(PipelineStep): MetabolomicsAnalysis, Metatranscriptome, ] -OmicsProcessing.update_forward_refs() -Biosample.update_forward_refs() -MAGCreate.update_forward_refs() +OmicsProcessing.model_rebuild() +Biosample.model_rebuild() +MAGCreate.model_rebuild() class FileDownloadMetadata(BaseModel): @@ -640,7 +631,7 @@ class EnvoTreeNode(BaseModel): children: List[EnvoTreeNode] -EnvoTreeNode.update_forward_refs() +EnvoTreeNode.model_rebuild() class EnvoTreeResponse(BaseModel): diff --git a/nmdc_server/schemas_submission.py b/nmdc_server/schemas_submission.py index d80144cc..9329e5ac 100644 --- a/nmdc_server/schemas_submission.py +++ b/nmdc_server/schemas_submission.py @@ -3,7 +3,7 @@ from typing import Any, Dict, List, Optional from uuid import UUID -from pydantic import BaseModel, ConfigDict, validator +from pydantic import BaseModel, ConfigDict, ValidationInfo, field_validator from nmdc_server import schemas from nmdc_server.models import SubmissionEditorRole @@ -121,15 +121,12 @@ class SubmissionMetadataSchema(SubmissionMetadataSchemaCreate): permission_level: Optional[str] = None model_config = ConfigDict(from_attributes=True) - # TODO[pydantic]: We couldn't refactor the `validator`, please replace it by `field_validator` - # manually. - # Check https://docs.pydantic.dev/dev-v2/migration/#changes-to-validators for more information. - @validator("metadata_submission", pre=True, always=True) - def populate_roles(cls, metadata_submission, values): - owners = set(values.get("owners", [])) - editors = set(values.get("editors", [])) - viewers = set(values.get("viewers", [])) - metadata_contributors = set(values.get("metadata_contributors", [])) + @field_validator("metadata_submission", mode="before") + def populate_roles(cls, metadata_submission, info: ValidationInfo): + owners = set(info.data.get("owners", [])) + editors = set(info.data.get("editors", [])) + viewers = set(info.data.get("viewers", [])) + metadata_contributors = set(info.data.get("metadata_contributors", [])) for contributor in metadata_submission.get("studyForm", {}).get("contributors", []): orcid = contributor.get("orcid", None) @@ -145,7 +142,7 @@ def populate_roles(cls, metadata_submission, values): return metadata_submission -SubmissionMetadataSchema.update_forward_refs() +SubmissionMetadataSchema.model_rebuild() class MetadataSuggestionRequest(BaseModel): From efc6481e7150066539136ef013bafb60b1a6b48a Mon Sep 17 00:00:00 2001 From: naglepuff Date: Wed, 16 Oct 2024 14:43:04 -0400 Subject: [PATCH 10/14] Only add debug toolbar in development mode --- nmdc_server/app.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/nmdc_server/app.py b/nmdc_server/app.py index 784c66e6..91814e78 100644 --- a/nmdc_server/app.py +++ b/nmdc_server/app.py @@ -52,7 +52,8 @@ async def lifespan(app: FastAPI): debug=True, lifespan=lifespan, ) - app.add_middleware(DebugToolbarMiddleware) + if settings.environment == "development": + app.add_middleware(DebugToolbarMiddleware) @app.get("/docs", response_class=RedirectResponse, status_code=301, include_in_schema=False) async def redirect_docs(): From 41f49af72185d4e41ffec18b525e6dab1a329d9e Mon Sep 17 00:00:00 2001 From: naglepuff Date: Thu, 17 Oct 2024 14:59:38 -0400 Subject: [PATCH 11/14] Use env var for debug mode --- nmdc_server/app.py | 2 +- nmdc_server/config.py | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/nmdc_server/app.py b/nmdc_server/app.py index 91814e78..b962ef7e 100644 --- a/nmdc_server/app.py +++ b/nmdc_server/app.py @@ -49,7 +49,7 @@ async def lifespan(app: FastAPI): version=__version__, docs_url="/api/docs", openapi_url="/api/openapi.json", - debug=True, + debug=settings.debug, lifespan=lifespan, ) if settings.environment == "development": diff --git a/nmdc_server/config.py b/nmdc_server/config.py index c36fa912..ea735a24 100644 --- a/nmdc_server/config.py +++ b/nmdc_server/config.py @@ -8,6 +8,7 @@ class Settings(BaseSettings): environment: str = "production" + debug: bool = False # Several different database urls are configured for different # environments. In production, only database_uri and ingest_database_uri From 85f816608d474a4a6af72bd0bf615cdca547e67b Mon Sep 17 00:00:00 2001 From: Michael Nagler Date: Thu, 17 Oct 2024 15:16:23 -0400 Subject: [PATCH 12/14] Import LoggingIntegration --- nmdc_server/app.py | 1 + 1 file changed, 1 insertion(+) diff --git a/nmdc_server/app.py b/nmdc_server/app.py index b962ef7e..dc46daef 100644 --- a/nmdc_server/app.py +++ b/nmdc_server/app.py @@ -7,6 +7,7 @@ from fastapi.middleware.cors import CORSMiddleware from fastapi.responses import RedirectResponse from fastapi.staticfiles import StaticFiles +from sentry_sdk.integrations.logging import LoggingIntegration from sentry_sdk.integrations.sqlalchemy import SqlalchemyIntegration from starlette.middleware.sessions import SessionMiddleware From 44c548681208e883013bcd1f10530e4e6370d2d6 Mon Sep 17 00:00:00 2001 From: Michael Nagler Date: Thu, 17 Oct 2024 15:16:45 -0400 Subject: [PATCH 13/14] Add LoggingIntegration Co-authored-by: Dan LaManna --- nmdc_server/app.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/nmdc_server/app.py b/nmdc_server/app.py index dc46daef..b284adbb 100644 --- a/nmdc_server/app.py +++ b/nmdc_server/app.py @@ -22,8 +22,12 @@ def attach_sentry(app: FastAPI): sentry_sdk.init( dsn=settings.sentry_dsn, - integrations=[SqlalchemyIntegration()], - enable_tracing=settings.sentry_tracing_enabled, + integrations=[ + LoggingIntegration(level=logging.INFO, event_level=logging.WARNING), + SqlalchemyIntegration(), + ], + in_app_include=["nmdc_server"], + attach_stacktrace=True, traces_sample_rate=settings.sentry_traces_sample_rate, ) From 9920a9f546e1216bd5418740e58083f3edfc2b08 Mon Sep 17 00:00:00 2001 From: naglepuff Date: Thu, 17 Oct 2024 15:29:23 -0400 Subject: [PATCH 14/14] Import logging, fix formatting --- nmdc_server/app.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/nmdc_server/app.py b/nmdc_server/app.py index b284adbb..7db4ec86 100644 --- a/nmdc_server/app.py +++ b/nmdc_server/app.py @@ -1,3 +1,4 @@ +import logging import typing from contextlib import asynccontextmanager @@ -23,8 +24,8 @@ def attach_sentry(app: FastAPI): sentry_sdk.init( dsn=settings.sentry_dsn, integrations=[ - LoggingIntegration(level=logging.INFO, event_level=logging.WARNING), - SqlalchemyIntegration(), + LoggingIntegration(level=logging.INFO, event_level=logging.WARNING), + SqlalchemyIntegration(), ], in_app_include=["nmdc_server"], attach_stacktrace=True,