diff --git a/docs/changelog.md b/docs/changelog.md index 2daa71f..5e88e58 100644 --- a/docs/changelog.md +++ b/docs/changelog.md @@ -2,6 +2,11 @@ This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html) and [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) format. +## [0.7.0] -- 2023-XX-XX +- Added pop to project table and annotation model +- Switched to pydantic2 +- Updated requirements (psycopg2 -> psycopg3) + ## [0.6.0] -- 2023-08-24 - Added date filter to project annotation diff --git a/pepdbagent/_version.py b/pepdbagent/_version.py index 906d362..c4b6bd4 100644 --- a/pepdbagent/_version.py +++ b/pepdbagent/_version.py @@ -1 +1 @@ -__version__ = "0.6.0" +__version__ = "0.7.0a1" diff --git a/pepdbagent/db_utils.py b/pepdbagent/db_utils.py index e758ea1..9cf5d52 100644 --- a/pepdbagent/db_utils.py +++ b/pepdbagent/db_utils.py @@ -1,11 +1,10 @@ import datetime import logging -from typing import Any, Optional, List +from typing import Optional, List from sqlalchemy import ( BigInteger, FetchedValue, - PrimaryKeyConstraint, Result, Select, String, @@ -13,7 +12,6 @@ select, TIMESTAMP, ForeignKey, - ForeignKeyConstraint, UniqueConstraint, ) from sqlalchemy.dialects.postgresql import JSON @@ -92,6 +90,7 @@ class Projects(Base): onupdate=deliver_update_date, default=deliver_update_date ) pep_schema: Mapped[Optional[str]] + pop: Mapped[Optional[bool]] = mapped_column(default=False) samples_mapping: Mapped[List["Samples"]] = relationship( back_populates="sample_mapping", cascade="all, delete-orphan" ) diff --git a/pepdbagent/models.py b/pepdbagent/models.py index 0703940..6ab60a7 100644 --- a/pepdbagent/models.py +++ b/pepdbagent/models.py @@ -1,9 +1,6 @@ # file with pydantic models -import datetime from typing import List, Optional, Union - -import peppy -from pydantic import BaseModel, Extra, Field, validator +from pydantic import BaseModel, Extra, Field, validator, ConfigDict, field_validator class AnnotationModel(BaseModel): @@ -21,12 +18,14 @@ class AnnotationModel(BaseModel): submission_date: Optional[str] digest: Optional[str] pep_schema: Optional[str] + pop: Optional[bool] = False - class Config: - allow_population_by_field_name = True - validate_assignment = True + model_config = ConfigDict( + validate_assignment=True, + populate_by_name=True, + ) - @validator("is_private") + @field_validator("is_private") def is_private_should_be_bool(cls, v): if not isinstance(v, bool): return False @@ -71,20 +70,20 @@ class UpdateItems(BaseModel): Model used for updating individual items in db """ - name: Optional[str] - description: Optional[str] - tag: Optional[str] - is_private: Optional[bool] - pep_schema: Optional[str] - digest: Optional[str] - config: Optional[dict] - samples: Optional[List[dict]] - subsamples: Optional[List[List[dict]]] - description: Optional[str] - - class Config: - arbitrary_types_allowed = True - extra = Extra.forbid + name: Optional[str] = None + description: Optional[str] = None + tag: Optional[str] = None + is_private: Optional[bool] = None + pep_schema: Optional[str] = None + digest: Optional[str] = None + config: Optional[dict] = None + samples: Optional[List[dict]] = None + subsamples: Optional[List[List[dict]]] = None + + model_config = ConfigDict( + arbitrary_types_allowed=True, + extra="forbid", + ) @property def number_of_samples(self) -> Union[int, None]: @@ -99,37 +98,35 @@ class UpdateModel(BaseModel): Model used for updating individual items and creating sql string in the code """ - config: Optional[dict] + config: Optional[dict] = None name: Optional[str] = None tag: Optional[str] = None - private: Optional[bool] = Field(alias="is_private") - digest: Optional[str] - number_of_samples: Optional[int] - pep_schema: Optional[str] + private: Optional[bool] = Field(alias="is_private", default=None) + digest: Optional[str] = None + number_of_samples: Optional[int] = None + pep_schema: Optional[str] = None description: Optional[str] = "" # last_update_date: Optional[datetime.datetime] = datetime.datetime.now(datetime.timezone.utc) - @validator("tag", "name") + @field_validator("tag", "name") def value_must_not_be_empty(cls, v): if "" == v: return None return v - @validator("tag", "name") + @field_validator("tag", "name") def value_must_be_lowercase(cls, v): if v: return v.lower() return v - @validator("tag", "name") + @field_validator("tag", "name") def value_should_not_contain_question(cls, v): if "?" in v: return ValueError("Question mark (?) is prohibited in name and tag.") return v - class Config: - extra = Extra.forbid - allow_population_by_field_name = True + model_config = ConfigDict(populate_by_name=True, extra="forbid") class NamespaceInfo(BaseModel): diff --git a/pepdbagent/modules/annotation.py b/pepdbagent/modules/annotation.py index 3b7873c..579406f 100644 --- a/pepdbagent/modules/annotation.py +++ b/pepdbagent/modules/annotation.py @@ -2,8 +2,7 @@ from datetime import datetime from typing import List, Literal, Optional, Union -from sqlalchemy import Engine, and_, func, or_, select -from sqlalchemy.exc import IntegrityError +from sqlalchemy import and_, func, or_, select from sqlalchemy.sql.selectable import Select from pepdbagent.const import ( @@ -189,6 +188,7 @@ def _get_single_annotation( Projects.last_update_date, Projects.digest, Projects.pep_schema, + Projects.pop, ).where( and_( Projects.name == name, @@ -214,6 +214,7 @@ def _get_single_annotation( last_update_date=str(query_result.last_update_date), digest=query_result.digest, pep_schema=query_result.pep_schema, + pop=query_result.pop, ) _LOGGER.info(f"Annotation of the project '{namespace}/{name}:{tag}' has been found!") return annot @@ -307,6 +308,7 @@ def _get_projects( Projects.last_update_date, Projects.digest, Projects.pep_schema, + Projects.pop, ).select_from(Projects) statement = self._add_condition( @@ -337,6 +339,7 @@ def _get_projects( last_update_date=str(result.last_update_date), digest=result.digest, pep_schema=result.pep_schema, + pop=result.pop, ) ) return results_list @@ -445,7 +448,7 @@ def _add_date_filter_if_provided( return statement else: if filter_by: - _LOGGER.warning(f"filter_start_date was not provided, skipping filter...") + _LOGGER.warning("filter_start_date was not provided, skipping filter...") return statement def get_project_number_in_namespace( diff --git a/pepdbagent/modules/project.py b/pepdbagent/modules/project.py index 6bb8b05..d3545dd 100644 --- a/pepdbagent/modules/project.py +++ b/pepdbagent/modules/project.py @@ -4,14 +4,20 @@ from typing import Union, List, NoReturn import peppy -from sqlalchemy import Engine, and_, delete, insert, or_, select, update +from sqlalchemy import and_, delete, select from sqlalchemy.exc import IntegrityError, NoResultFound from sqlalchemy.orm import Session from sqlalchemy import Select from peppy.const import SAMPLE_RAW_DICT_KEY, SUBSAMPLE_RAW_LIST_KEY, CONFIG_KEY -from pepdbagent.const import * +from pepdbagent.const import ( + DEFAULT_TAG, + DESCRIPTION_KEY, + NAME_KEY, + PKG_NAME, +) + from pepdbagent.db_utils import Projects, Samples, Subsamples, BaseEngine from pepdbagent.exceptions import ProjectNotFoundError, ProjectUniqueNameError from pepdbagent.models import UpdateItems, UpdateModel @@ -200,6 +206,7 @@ def create( tag: str = DEFAULT_TAG, description: str = None, is_private: bool = False, + pop: bool = False, pep_schema: str = None, overwrite: bool = False, update_only: bool = False, @@ -214,7 +221,8 @@ def create( :param name: name of the project (Default: name is taken from the project object) :param tag: tag (or version) of the project. :param is_private: boolean value if the project should be visible just for user that creates it. - :param pep_schema: assign PEP to a specific schema. [DefaultL: None] + :param pep_schema: assign PEP to a specific schema. [Default: None] + :param pop: if project is a pep of peps (POP) [Default: False] :param overwrite: if project exists overwrite the project, otherwise upload it. [Default: False - project won't be overwritten if it exists in db] :param update_only: if project exists overwrite it, otherwise do nothing. [Default: False] @@ -232,7 +240,7 @@ def create( elif proj_dict[CONFIG_KEY][NAME_KEY]: proj_name = proj_dict[CONFIG_KEY][NAME_KEY].lower() else: - raise ValueError(f"Name of the project wasn't provided. Project will not be uploaded.") + raise ValueError("Name of the project wasn't provided. Project will not be uploaded.") proj_dict[CONFIG_KEY][NAME_KEY] = proj_name @@ -251,6 +259,7 @@ def create( private=is_private, pep_schema=pep_schema, description=description, + pop=pop, ) return None else: @@ -268,6 +277,7 @@ def create( last_update_date=datetime.datetime.now(datetime.timezone.utc), pep_schema=pep_schema, description=description, + pop=pop, ) self._add_samples_to_project(new_prj, proj_dict[SAMPLE_RAW_DICT_KEY]) @@ -299,9 +309,9 @@ def create( else: raise ProjectUniqueNameError( - f"Namespace, name and tag already exists. Project won't be " - f"uploaded. Solution: Set overwrite value as True" - f" (project will be overwritten), or change tag!" + "Namespace, name and tag already exists. Project won't be " + "uploaded. Solution: Set overwrite value as True" + " (project will be overwritten), or change tag!" ) def _overwrite( @@ -315,6 +325,7 @@ def _overwrite( private: bool = False, pep_schema: str = None, description: str = "", + pop: bool = False, ) -> None: """ Update existing project by providing all necessary information. @@ -328,6 +339,7 @@ def _overwrite( :param private: boolean value if the project should be visible just for user that creates it. :param pep_schema: assign PEP to a specific schema. [DefaultL: None] :param description: project description + :param pop: if project is a pep of peps, simply POP [Default: False] :return: None """ proj_name = proj_name.lower() @@ -351,6 +363,7 @@ def _overwrite( found_prj.config = project_dict[CONFIG_KEY] found_prj.description = description found_prj.last_update_date = datetime.datetime.now(datetime.timezone.utc) + found_prj.pop = pop # Deleting old samples and subsamples if found_prj.samples_mapping: @@ -476,14 +489,14 @@ def __create_update_dict(update_values: UpdateItems) -> dict: updating values :return: unified update dict """ - update_final = UpdateModel() + update_final = UpdateModel.model_construct() if update_values.name is not None: if update_values.config is not None: update_values.config[NAME_KEY] = update_values.name update_final = UpdateModel( name=update_values.name, - **update_final.dict(exclude_unset=True), + **update_final.model_dump(exclude_unset=True), ) if update_values.description is not None: @@ -491,49 +504,49 @@ def __create_update_dict(update_values: UpdateItems) -> dict: update_values.config[DESCRIPTION_KEY] = update_values.description update_final = UpdateModel( description=update_values.description, - **update_final.dict(exclude_unset=True), + **update_final.model_dump(exclude_unset=True), ) if update_values.config is not None: update_final = UpdateModel( - config=update_values.config, **update_final.dict(exclude_unset=True) + config=update_values.config, **update_final.model_dump(exclude_unset=True) ) name = update_values.config.get(NAME_KEY) description = update_values.config.get(DESCRIPTION_KEY) if name: update_final = UpdateModel( name=name, - **update_final.dict(exclude_unset=True, exclude={NAME_KEY}), + **update_final.model_dump(exclude_unset=True, exclude={NAME_KEY}), ) if description: update_final = UpdateModel( description=description, - **update_final.dict(exclude_unset=True, exclude={DESCRIPTION_KEY}), + **update_final.model_dump(exclude_unset=True, exclude={DESCRIPTION_KEY}), ) if update_values.tag is not None: update_final = UpdateModel( - tag=update_values.tag, **update_final.dict(exclude_unset=True) + tag=update_values.tag, **update_final.model_dump(exclude_unset=True) ) if update_values.is_private is not None: update_final = UpdateModel( is_private=update_values.is_private, - **update_final.dict(exclude_unset=True), + **update_final.model_dump(exclude_unset=True), ) if update_values.pep_schema is not None: update_final = UpdateModel( pep_schema=update_values.pep_schema, - **update_final.dict(exclude_unset=True), + **update_final.model_dump(exclude_unset=True), ) if update_values.number_of_samples is not None: update_final = UpdateModel( number_of_samples=update_values.number_of_samples, - **update_final.dict(exclude_unset=True), + **update_final.model_dump(exclude_unset=True), ) - return update_final.dict(exclude_unset=True, exclude_none=True) + return update_final.model_dump(exclude_unset=True, exclude_none=True) def exists( self, @@ -565,7 +578,7 @@ def exists( return False @staticmethod - def _add_samples_to_project(projects_sa: Projects, samples: List[dict]) -> NoReturn: + def _add_samples_to_project(projects_sa: Projects, samples: List[dict]) -> None: """ Add samples to the project sa object. (With commit this samples will be added to the 'samples table') :param projects_sa: Projects sa object, in open session @@ -575,6 +588,8 @@ def _add_samples_to_project(projects_sa: Projects, samples: List[dict]) -> NoRet for row_number, sample in enumerate(samples): projects_sa.samples_mapping.append(Samples(sample=sample, row_number=row_number)) + return None + @staticmethod def _add_subsamples_to_project( projects_sa: Projects, subsamples: List[List[dict]] diff --git a/pepdbagent/utils.py b/pepdbagent/utils.py index dc11bcc..c58cff4 100644 --- a/pepdbagent/utils.py +++ b/pepdbagent/utils.py @@ -7,7 +7,7 @@ import ubiquerg from peppy.const import SAMPLE_RAW_DICT_KEY -from .exceptions import IncorrectDateFormat, RegistryPathError +from .exceptions import RegistryPathError def is_valid_registry_path(rpath: str) -> bool: diff --git a/requirements/requirements-all.txt b/requirements/requirements-all.txt index 0b8fbeb..c658ca1 100644 --- a/requirements/requirements-all.txt +++ b/requirements/requirements-all.txt @@ -4,5 +4,5 @@ peppy>=0.40.0a4 ubiquerg>=0.6.2 coloredlogs>=15.0.1 pytest-mock -pydantic<2.0 -psycopg2-binary +pydantic>=2.0 +psycopg diff --git a/tests/README.md b/tests/README.md index b0dd607..b057606 100644 --- a/tests/README.md +++ b/tests/README.md @@ -3,13 +3,12 @@ ### How to run tests localy: 1. Use or create empty database with next credentials: ```txt -POSTGRES_USER=postgres -POSTGRES_PASSWORD=docker -POSTGRES_DB=pep-db -POSTGRES_PORT=5432 +docker run --rm -it --name bedbase \ + -e POSTGRES_USER=postgres \ + -e POSTGRES_PASSWORD=docker \ + -e POSTGRES_DB=pep-db \ + -p 5432:5432 postgres ``` -Database can be created using docker file: [../pep_db/Dockerfile](../pep_db/Dockerfile) -To run docker use this tutorial [../docs/db_tutorial.md](../docs/db_tutorial.md) 2. Run pytest using this command: `pytest` diff --git a/tests/conftest.py b/tests/conftest.py index f210dbf..a99d059 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -5,9 +5,10 @@ from sqlalchemy import create_engine from sqlalchemy import text -DNS = f"postgresql://postgres:docker@localhost:5432/pep-db" from pepdbagent import PEPDatabaseAgent +DNS = f"postgresql+psycopg://postgres:docker@localhost:5432/pep-db" + DATA_PATH = os.path.join( os.path.dirname(os.path.dirname(os.path.abspath(__file__))), diff --git a/tests/test_pepagent.py b/tests/test_pepagent.py index 7a5e84a..d299bdf 100644 --- a/tests/test_pepagent.py +++ b/tests/test_pepagent.py @@ -5,8 +5,7 @@ import pytest from pepdbagent.exceptions import FilterError, ProjectNotFoundError - -DNS = f"postgresql://postgres:docker@localhost:5432/pep-db" +from .conftest import DNS DATA_PATH = os.path.join( @@ -386,7 +385,7 @@ def test_all_annotations_are_returned(self, initiate_pepdb_con, namespace, name) name=name, tag="default", ) - assert result.results[0].__fields_set__ == { + assert result.results[0].model_fields_set == { "is_private", "tag", "namespace", @@ -397,6 +396,7 @@ def test_all_annotations_are_returned(self, initiate_pepdb_con, namespace, name) "last_update_date", "submission_date", "pep_schema", + "pop", } @pytest.mark.parametrize(