From 3bbba45f165085d2e310f6d5e36bc373ee15b36e Mon Sep 17 00:00:00 2001 From: Khoroshevskyi Date: Tue, 9 Jan 2024 12:11:41 -0500 Subject: [PATCH 1/6] work on views --- pepdbagent/db_utils.py | 42 +++++++++++++++ pepdbagent/modules/view.py | 103 +++++++++++++++++++++++++++++++++++++ pepdbagent/pepdbagent.py | 6 +++ tests/conftest.py | 3 ++ tests/test_pepagent.py | 68 ++++++++++++++++++++++++ 5 files changed, 222 insertions(+) create mode 100644 pepdbagent/modules/view.py diff --git a/pepdbagent/db_utils.py b/pepdbagent/db_utils.py index f6ed227..ccf76b1 100644 --- a/pepdbagent/db_utils.py +++ b/pepdbagent/db_utils.py @@ -100,6 +100,10 @@ class Projects(Base): stars_mapping: Mapped[List["Stars"]] = relationship( back_populates="project_mapping", cascade="all, delete-orphan" ) + views_mapping: Mapped[List["Views"]] = relationship( + back_populates="project_mapping", cascade="all, delete-orphan" + ) + __table_args__ = (UniqueConstraint("namespace", "name", "tag"),) @@ -117,6 +121,10 @@ class Samples(Base): sample_name: Mapped[Optional[str]] = mapped_column() sample_mapping: Mapped["Projects"] = relationship(back_populates="samples_mapping") + views: Mapped[List["ViewSampleAssociation"]] = relationship( + back_populates="sample", cascade="all, delete-orphan" + ) + class Subsamples(Base): """ @@ -160,6 +168,40 @@ class Stars(Base): project_mapping: Mapped["Projects"] = relationship(back_populates="stars_mapping") +class Views(Base): + """ + Views table representation in the database + """ + + __tablename__ = "views" + + id: Mapped[int] = mapped_column(primary_key=True) + name: Mapped[str] = mapped_column() + description: Mapped[Optional[str]] + + project_id = mapped_column(ForeignKey("projects.id", ondelete="CASCADE")) + project_mapping = relationship("Projects", back_populates="views_mapping") + + samples: Mapped[List["ViewSampleAssociation"]] = relationship( + back_populates="view", cascade="all, delete-orphan" + ) + + _table_args__ = (UniqueConstraint("namespace", "project_id"),) + + +class ViewSampleAssociation(Base): + """ + Association table between views and samples + """ + + __tablename__ = "views_samples" + + sample_id = mapped_column(ForeignKey("samples.id", ondelete="CASCADE"), primary_key=True) + view_id = mapped_column(ForeignKey("views.id", ondelete="CASCADE"), primary_key=True) + sample: Mapped["Samples"] = relationship(back_populates="views") + view: Mapped["Views"] = relationship(back_populates="samples") + + class BaseEngine: """ A class with base methods, that are used in several classes. e.g. fetch_one or fetch_all diff --git a/pepdbagent/modules/view.py b/pepdbagent/modules/view.py new file mode 100644 index 0000000..d4ebc42 --- /dev/null +++ b/pepdbagent/modules/view.py @@ -0,0 +1,103 @@ +# View of the PEP. In other words, it is a part of the PEP, or subset of the samples in the PEP. + +import logging +from typing import Union +import datetime + +import peppy +from peppy.const import SAMPLE_TABLE_INDEX_KEY +from sqlalchemy import select, and_, func +from sqlalchemy.orm import Session +from sqlalchemy.orm.attributes import flag_modified + + +from pepdbagent.const import ( + DEFAULT_TAG, + PKG_NAME, +) +from pepdbagent.exceptions import SampleNotFoundError + +from pepdbagent.db_utils import BaseEngine, Samples, Projects + +_LOGGER = logging.getLogger(PKG_NAME) + + +class PEPDatabaseView: + """ + Class that represents Project in Database. + + While using this class, user can create, retrieve, delete, and update projects from database + """ + + def __init__(self, pep_db_engine: BaseEngine): + """ + :param pep_db_engine: pepdbengine object with sa engine + """ + self._sa_engine = pep_db_engine.engine + self._pep_db_engine = pep_db_engine + + def get( + self, + namespace: str, + name: str, + tag: str = DEFAULT_TAG, + raw: bool = False, + ) -> Union[peppy.Project, dict, None]: + """ + Retrieve view of the project from the database. + View is a subset of the samples in the project. e.g. bed-db project has all the samples in bedbase, + bedset is a view of the bedbase project with only the samples in the bedset. + + :param namespace: namespace of the project + :param name: name of the project (Default: name is taken from the project object) + :param raw: retrieve unprocessed (raw) PEP dict. + :return: peppy.Project object with found project or dict with unprocessed + PEP elements: { + name: str + description: str + _config: dict + _sample_dict: dict + _subsample_dict: dict + } + """ + ... + + def create( + self, + namespace: str, + view_name: str, + sample_id_list: Union[list, tuple, str] = None, + description: str = None, + ): + ... + + def delete(self): + ... + + def add_sample( + self, + namespace: str, + name: str, + tag: str, + sample_name: str, + view_name: str, + ): + ... + + def remove_sample(self, namespace: str, view_name: str, sample_name: str): + ... + + def get_snap_view( + self, namespace: str, name: str, tag: str, sample_name_list: Union[list, tuple, str] = None + ): + """ + Get a snap view of the project. Snap view is a view of the project + with only the samples in the list. This view won't be saved in the database. + + :param namespace: + :param name: + :param tag: + :param sample_name_list: + :return: + """ + ... diff --git a/pepdbagent/pepdbagent.py b/pepdbagent/pepdbagent.py index 74dddf4..19995e5 100644 --- a/pepdbagent/pepdbagent.py +++ b/pepdbagent/pepdbagent.py @@ -5,6 +5,7 @@ from pepdbagent.modules.project import PEPDatabaseProject from pepdbagent.modules.user import PEPDatabaseUser from pepdbagent.modules.sample import PEPDatabaseSample +from pepdbagent.modules.view import PEPDatabaseView class PEPDatabaseAgent(object): @@ -51,6 +52,7 @@ def __init__( self.__namespace = PEPDatabaseNamespace(pep_db_engine) self.__sample = PEPDatabaseSample(pep_db_engine) self.__user = PEPDatabaseUser(pep_db_engine) + self.__view = PEPDatabaseView(pep_db_engine) self.__db_name = database @@ -74,6 +76,10 @@ def user(self) -> PEPDatabaseUser: def sample(self) -> PEPDatabaseSample: return self.__sample + @property + def view(self) -> PEPDatabaseView: + return self.__view + def __str__(self): return f"Connection to the database: '{self.__db_name}' is set!" diff --git a/tests/conftest.py b/tests/conftest.py index 70ec847..32d3dbe 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -42,6 +42,9 @@ def initiate_pepdb_con( conn.execute(text("DROP table IF EXISTS subsamples CASCADE")) conn.execute(text("DROP table IF EXISTS stars CASCADE")) conn.execute(text("DROP table IF EXISTS users CASCADE")) + conn.execute(text("DROP table IF EXISTS views CASCADE")) + conn.execute(text("DROP table IF EXISTS views_samples CASCADE")) + pepdb_con = PEPDatabaseAgent(dsn=DNS, echo=True) for namespace, item in list_of_available_peps.items(): if namespace == "private_test": diff --git a/tests/test_pepagent.py b/tests/test_pepagent.py index 7af269d..58b743d 100644 --- a/tests/test_pepagent.py +++ b/tests/test_pepagent.py @@ -898,3 +898,71 @@ def test_delete_and_add(self, initiate_pepdb_con, namespace, name, tag, sample_d initiate_pepdb_con.sample.add(namespace, name, tag, sample_dict) prj2 = initiate_pepdb_con.project.get(namespace, name) assert prj.get_sample("pig_0h").to_dict() == prj2.get_sample("pig_0h").to_dict() + + +# @pytest.mark.skipif( +# not db_setup(), +# reason="DB is not setup", +# ) +@pytest.mark.skipif( + True, + reason="Not implemented", +) +class TestViews: + """ + Test function within view class + """ + + @pytest.mark.parametrize( + "namespace, name, sample_name", + [ + ["namespace1", "amendments1", "pig_0h"], + ], + ) + def test_create_view(self, initiate_pepdb_con, namespace, name, sample_name): + ... + + @pytest.mark.parametrize( + "namespace, name, sample_name", + [ + ["namespace1", "amendments1", "pig_0h"], + ], + ) + def test_delete_view(self, initiate_pepdb_con, namespace, name, sample_name): + ... + + @pytest.mark.parametrize( + "namespace, name, sample_name", + [ + ["namespace1", "amendments1", "pig_0h"], + ], + ) + def test_add_sample_to_view(self, initiate_pepdb_con, namespace, name, sample_name): + ... + + @pytest.mark.parametrize( + "namespace, name, sample_name", + [ + ["namespace1", "amendments1", "pig_0h"], + ], + ) + def test_remove_sample_from_view(self, initiate_pepdb_con, namespace, name, sample_name): + ... + + @pytest.mark.parametrize( + "namespace, name, sample_name", + [ + ["namespace1", "amendments1", "pig_0h"], + ], + ) + def test_add_existing_sample_in_view(self, initiate_pepdb_con, namespace, name, sample_name): + ... + + @pytest.mark.parametrize( + "namespace, name, sample_name", + [ + ["namespace1", "amendments1", "pig_0h"], + ], + ) + def test_get_snap_view(self, initiate_pepdb_con, namespace, name, sample_name): + ... From 6e944e049ef720536eb41c4345e34b8bd6da2310 Mon Sep 17 00:00:00 2001 From: Khoroshevskyi Date: Tue, 9 Jan 2024 23:31:54 -0500 Subject: [PATCH 2/6] Added views module --- pepdbagent/__init__.py | 7 +- pepdbagent/db_utils.py | 2 +- pepdbagent/exceptions.py | 14 ++ pepdbagent/models.py | 24 +++ pepdbagent/modules/view.py | 320 ++++++++++++++++++++++++++++++++++--- tests/test_pepagent.py | 119 +++++++++++--- 6 files changed, 439 insertions(+), 47 deletions(-) diff --git a/pepdbagent/__init__.py b/pepdbagent/__init__.py index 4098c3b..11e033b 100644 --- a/pepdbagent/__init__.py +++ b/pepdbagent/__init__.py @@ -2,8 +2,11 @@ import coloredlogs import logmuse -from ._version import __version__ -from .pepdbagent import * +from pepdbagent._version import __version__ +from pepdbagent.pepdbagent import PEPDatabaseAgent + +__all__ = ["__version__", "PEPDatabaseAgent"] + _LOGGER = logmuse.init_logger("pepdbagent") coloredlogs.install( diff --git a/pepdbagent/db_utils.py b/pepdbagent/db_utils.py index 398b6ac..8b4251b 100644 --- a/pepdbagent/db_utils.py +++ b/pepdbagent/db_utils.py @@ -137,7 +137,7 @@ class Samples(Base): sample_name: Mapped[Optional[str]] = mapped_column() sample_mapping: Mapped["Projects"] = relationship(back_populates="samples_mapping") - views: Mapped[List["ViewSampleAssociation"]] = relationship( + views: Mapped[Optional[List["ViewSampleAssociation"]]] = relationship( back_populates="sample", cascade="all, delete-orphan" ) diff --git a/pepdbagent/exceptions.py b/pepdbagent/exceptions.py index 30a2b12..462301e 100644 --- a/pepdbagent/exceptions.py +++ b/pepdbagent/exceptions.py @@ -59,3 +59,17 @@ def __init__(self, msg=""): class SampleNotFoundError(PEPDatabaseAgentError): def __init__(self, msg=""): super().__init__(f"""Sample does not exist. {msg}""") + + +class ViewNotFoundError(PEPDatabaseAgentError): + def __init__(self, msg=""): + super().__init__(f"""View does not exist. {msg}""") + + +class SampleAlreadyInView(PEPDatabaseAgentError): + """ + Sample is already in the view exception + """ + + def __init__(self, msg=""): + super().__init__(f"""Sample is already in the view. {msg}""") diff --git a/pepdbagent/models.py b/pepdbagent/models.py index dcf2d31..eb2116d 100644 --- a/pepdbagent/models.py +++ b/pepdbagent/models.py @@ -160,3 +160,27 @@ class ProjectRegistryPath(BaseModel): namespace: str name: str tag: str = DEFAULT_TAG + + +class ViewAnnotation(BaseModel): + """ + View annotation model + """ + + project_namespace: str + project_name: str + project_tag: str + name: str + description: Optional[str] = None + number_of_samples: int + + +class CreateViewDictModel(BaseModel): + """ + View creation dict model + """ + + project_namespace: str + project_name: str + project_tag: str + sample_list: List[str] diff --git a/pepdbagent/modules/view.py b/pepdbagent/modules/view.py index d4ebc42..89c5dfa 100644 --- a/pepdbagent/modules/view.py +++ b/pepdbagent/modules/view.py @@ -1,23 +1,22 @@ # View of the PEP. In other words, it is a part of the PEP, or subset of the samples in the PEP. import logging -from typing import Union -import datetime +from typing import Union, List import peppy -from peppy.const import SAMPLE_TABLE_INDEX_KEY -from sqlalchemy import select, and_, func +from sqlalchemy import select, and_, delete from sqlalchemy.orm import Session -from sqlalchemy.orm.attributes import flag_modified +from sqlalchemy.exc import IntegrityError from pepdbagent.const import ( DEFAULT_TAG, PKG_NAME, ) -from pepdbagent.exceptions import SampleNotFoundError +from pepdbagent.exceptions import ViewNotFoundError, SampleAlreadyInView -from pepdbagent.db_utils import BaseEngine, Samples, Projects +from pepdbagent.db_utils import BaseEngine, Samples, Projects, Views, ViewSampleAssociation +from pepdbagent.models import ViewAnnotation, CreateViewDictModel _LOGGER = logging.getLogger(PKG_NAME) @@ -41,6 +40,7 @@ def get( namespace: str, name: str, tag: str = DEFAULT_TAG, + view_name: str = None, raw: bool = False, ) -> Union[peppy.Project, dict, None]: """ @@ -50,6 +50,8 @@ def get( :param namespace: namespace of the project :param name: name of the project (Default: name is taken from the project object) + :param tag: tag of the project (Default: tag is taken from the project object) + :param view_name: name of the view :param raw: retrieve unprocessed (raw) PEP dict. :return: peppy.Project object with found project or dict with unprocessed PEP elements: { @@ -60,44 +62,310 @@ def get( _subsample_dict: dict } """ - ... + view_statement = select(Views).where( + and_( + Views.project_mapping.has(namespace=namespace, name=name, tag=tag), + Views.name == view_name, + ) + ) + + with Session(self._sa_engine) as sa_session: + view = sa_session.scalar(view_statement) + if not view: + raise ViewNotFoundError( + f"View {view_name} of the project {namespace}/{name}:{tag} does not exist" + ) + samples = [sample.sample.sample for sample in view.samples] + config = view.project_mapping.config + sub_project_dict = {"_config": config, "_sample_dict": samples, "_subsample_dict": None} + if raw: + return sub_project_dict + else: + return peppy.Project.from_dict(sub_project_dict) + + def get_annotation( + self, namespace: str, name: str, tag: str = DEFAULT_TAG, view_name: str = None + ) -> ViewAnnotation: + """ + Get annotation of the view. + + :param namespace: namespace of the project + :param name: name of the project + :param tag: tag of the project + :param view_name: name of the sample + :return: ViewAnnotation object: + {project_namespace: str, + project_name: str, + project_tag: str, + name: str, + description: str, + number_of_samples: int} + """ + view_statement = select(Views).where( + and_( + Views.project_mapping.has(namespace=namespace, name=name, tag=tag), + Views.name == view_name, + ) + ) + + with Session(self._sa_engine) as sa_session: + view = sa_session.scalar(view_statement) + if not view: + raise ViewNotFoundError( + f"View {name} of the project {namespace}/{name}:{tag} does not exist" + ) + return ViewAnnotation( + project_namespace=namespace, + project_name=name, + project_tag=tag, + name=view.name, + description=view.description, + number_of_samples=len(view.samples), + ) def create( self, - namespace: str, view_name: str, - sample_id_list: Union[list, tuple, str] = None, + view_dict: Union[dict, CreateViewDictModel], description: str = None, - ): - ... + ) -> None: + """ + Create a view of the project in the database. + + :param view_name: namespace of the project + :param view_dict: dict or CreateViewDictModel object with view samples. + Dict should have the following structure: + { + project_namespace: str + project_name: str + project_tag: str + sample_list: List[str] # list of sample names + } + :param description: description of the view + retrun: None + """ + if isinstance(view_dict, dict): + view_dict = CreateViewDictModel(**view_dict) + + project_statement = select(Projects).where( + and_( + Projects.namespace == view_dict.project_namespace, + Projects.name == view_dict.project_name, + Projects.tag == view_dict.project_tag, + ) + ) + + with Session(self._sa_engine) as sa_session: + project = sa_session.scalar(project_statement) + if not project: + raise ValueError( + f"Project {view_dict.project_namespace}/{view_dict.project_name}:{view_dict.project_tag} does not exist" + ) + view = Views( + name=view_name, + description=description, + project_mapping=project, + ) + sa_session.add(view) - def delete(self): - ... + for sample_name in view_dict.sample_list: + sample_statement = select(Samples.id).where( + and_( + Samples.project_id == project.id, + Samples.sample_name == sample_name, + ) + ) + sample_id = sa_session.execute(sample_statement).one()[0] + if not sample_id: + raise ValueError( + f"Sample {view_dict.project_namespace}/{view_dict.project_name}:{view_dict.project_tag}:{sample_name} does not exist" + ) + sa_session.add(ViewSampleAssociation(sample_id=sample_id, view=view)) + + sa_session.commit() + + def delete( + self, + project_namespace: str, + project_name: str, + project_tag: str = DEFAULT_TAG, + view_name: str = None, + ) -> None: + """ + Delete a view of the project in the database. + + :param project_namespace: namespace of the project + :param project_name: name of the project + :param project_tag: tag of the project + :param view_name: name of the view + :return: None + """ + view_statement = select(Views).where( + and_( + Views.project_mapping.has( + namespace=project_namespace, name=project_name, tag=project_tag + ), + Views.name == view_name, + ) + ) + + with Session(self._sa_engine) as sa_session: + view = sa_session.scalar(view_statement) + if not view: + raise ViewNotFoundError( + f"View {view_name} of the project {project_namespace}/{project_name}:{project_tag} does not exist" + ) + sa_session.delete(view) + sa_session.commit() def add_sample( self, namespace: str, name: str, tag: str, - sample_name: str, view_name: str, + sample_name: Union[str, List[str]], ): - ... + """ + Add sample to the view. + + :param namespace: namespace of the project + :param name: name of the project + :param tag: tag of the project + :param view_name: name of the view + :param sample_name: sample name + :return: None + """ + if isinstance(sample_name, str): + sample_name = [sample_name] + view_statement = select(Views).where( + and_( + Views.project_mapping.has(namespace=namespace, name=name, tag=tag), + Views.name == view_name, + ) + ) + + with Session(self._sa_engine) as sa_session: + view = sa_session.scalar(view_statement) + if not view: + raise ViewNotFoundError( + f"View {view_name} of the project {namespace}/{name}:{tag} does not exist" + ) + for sample_name_one in sample_name: + sample_statement = select(Samples).where( + and_( + Samples.project_id == view.project_mapping.id, + Samples.sample_name == sample_name_one, + ) + ) + sample = sa_session.scalar(sample_statement) + if not sample: + raise ValueError( + f"Sample {namespace}/{name}:{tag}:{sample_name} does not exist" + ) + try: + sa_session.add(ViewSampleAssociation(sample=sample, view=view)) + sa_session.commit() + except IntegrityError: + raise SampleAlreadyInView( + f"Sample {namespace}/{name}:{tag}:{sample_name} already in view {view_name}" + ) - def remove_sample(self, namespace: str, view_name: str, sample_name: str): - ... + return None + + def remove_sample( + self, + namespace: str, + name: str, + tag: str, + view_name: str, + sample_name: str, + ) -> None: + """ + Remove sample from the view. + + :param namespace: namespace of the project + :param name: name of the project + :param tag: tag of the project + :param view_name: name of the view + :param sample_name: sample name + :return: None + """ + view_statement = select(Views).where( + and_( + Views.project_mapping.has(namespace=namespace, name=name, tag=tag), + Views.name == view_name, + ) + ) + + with Session(self._sa_engine) as sa_session: + view = sa_session.scalar(view_statement) + if not view: + raise ViewNotFoundError( + f"View {view_name} of the project {namespace}/{name}:{tag} does not exist" + ) + sample_statement = select(Samples).where( + and_( + Samples.project_id == view.project_mapping.id, + Samples.sample_name == sample_name, + ) + ) + sample = sa_session.scalar(sample_statement) + delete_statement = delete(ViewSampleAssociation).where( + and_( + ViewSampleAssociation.sample_id == sample.id, + ViewSampleAssociation.view_id == view.id, + ) + ) + sa_session.execute(delete_statement) + sa_session.commit() + + return None def get_snap_view( - self, namespace: str, name: str, tag: str, sample_name_list: Union[list, tuple, str] = None - ): + self, namespace: str, name: str, tag: str, sample_name_list: List[str], raw: bool = False + ) -> Union[peppy.Project, dict]: """ Get a snap view of the project. Snap view is a view of the project with only the samples in the list. This view won't be saved in the database. - :param namespace: - :param name: - :param tag: - :param sample_name_list: - :return: + :param namespace: project namespace + :param name: name of the project + :param tag: tag of the project + :param sample_name_list: list of sample names e.g. ["sample1", "sample2"] + :param raw: retrieve unprocessed (raw) PEP dict. + :return: peppy.Project object """ - ... + project_statement = select(Projects).where( + and_( + Projects.namespace == namespace, + Projects.name == name, + Projects.tag == tag, + ) + ) + with Session(self._sa_engine) as sa_session: + project = sa_session.scalar(project_statement) + if not project: + raise ValueError(f"Project {namespace}/{name}:{tag} does not exist") + samples = [] + for sample_name in sample_name_list: + sample_statement = select(Samples).where( + and_( + Samples.project_id == project.id, + Samples.sample_name == sample_name, + ) + ) + sample = sa_session.scalar(sample_statement) + if not sample: + raise ValueError( + f"Sample {namespace}/{name}:{tag}:{sample_name} does not exist" + ) + samples.append(sample.sample) + config = project.config + + if raw: + return {"_config": config, "_sample_dict": samples, "_subsample_dict": None} + else: + return peppy.Project.from_dict( + {"_config": config, "_sample_dict": samples, "_subsample_dict": None} + ) diff --git a/tests/test_pepagent.py b/tests/test_pepagent.py index 25b0c5d..5110b62 100644 --- a/tests/test_pepagent.py +++ b/tests/test_pepagent.py @@ -13,6 +13,8 @@ ProjectNotInFavorites, ProjectAlreadyInFavorites, SampleNotFoundError, + ViewNotFoundError, + SampleAlreadyInView, ) from .conftest import DNS @@ -1023,13 +1025,9 @@ def test_delete_and_add(self, initiate_pepdb_con, namespace, name, tag, sample_d assert prj.get_sample("pig_0h").to_dict() == prj2.get_sample("pig_0h").to_dict() -# @pytest.mark.skipif( -# not db_setup(), -# reason="DB is not setup", -# ) @pytest.mark.skipif( - True, - reason="Not implemented", + not db_setup(), + reason="DB is not setup", ) class TestViews: """ @@ -1037,13 +1035,26 @@ class TestViews: """ @pytest.mark.parametrize( - "namespace, name, sample_name", + "namespace, name, sample_name, view_name", [ - ["namespace1", "amendments1", "pig_0h"], + ["namespace1", "amendments1", "pig_0h", "view1"], ], ) - def test_create_view(self, initiate_pepdb_con, namespace, name, sample_name): - ... + def test_create_view(self, initiate_pepdb_con, namespace, name, sample_name, view_name): + initiate_pepdb_con.view.create( + view_name, + { + "project_namespace": namespace, + "project_name": name, + "project_tag": "default", + "sample_list": [sample_name, "pig_1h"], + }, + ) + + project = initiate_pepdb_con.project.get(namespace, name) + view_project = initiate_pepdb_con.view.get(namespace, name, "default", view_name) + assert len(view_project.samples) == 2 + assert view_project != project @pytest.mark.parametrize( "namespace, name, sample_name", @@ -1052,7 +1063,20 @@ def test_create_view(self, initiate_pepdb_con, namespace, name, sample_name): ], ) def test_delete_view(self, initiate_pepdb_con, namespace, name, sample_name): - ... + initiate_pepdb_con.view.create( + "view1", + { + "project_namespace": namespace, + "project_name": name, + "project_tag": "default", + "sample_list": [sample_name, "pig_1h"], + }, + ) + assert len(initiate_pepdb_con.view.get(namespace, name, "default", "view1").samples) == 2 + initiate_pepdb_con.view.delete(namespace, name, "default", "view1") + with pytest.raises(ViewNotFoundError): + initiate_pepdb_con.view.get(namespace, name, "default", "view1") + assert len(initiate_pepdb_con.project.get(namespace, name).samples) == 4 @pytest.mark.parametrize( "namespace, name, sample_name", @@ -1061,7 +1085,17 @@ def test_delete_view(self, initiate_pepdb_con, namespace, name, sample_name): ], ) def test_add_sample_to_view(self, initiate_pepdb_con, namespace, name, sample_name): - ... + initiate_pepdb_con.view.create( + "view1", + { + "project_namespace": namespace, + "project_name": name, + "project_tag": "default", + "sample_list": [sample_name], + }, + ) + initiate_pepdb_con.view.add_sample(namespace, name, "default", "view1", "pig_1h") + assert len(initiate_pepdb_con.view.get(namespace, name, "default", "view1").samples) == 2 @pytest.mark.parametrize( "namespace, name, sample_name", @@ -1069,8 +1103,20 @@ def test_add_sample_to_view(self, initiate_pepdb_con, namespace, name, sample_na ["namespace1", "amendments1", "pig_0h"], ], ) - def test_remove_sample_from_view(self, initiate_pepdb_con, namespace, name, sample_name): - ... + def test_add_multiple_samples_to_view(self, initiate_pepdb_con, namespace, name, sample_name): + initiate_pepdb_con.view.create( + "view1", + { + "project_namespace": namespace, + "project_name": name, + "project_tag": "default", + "sample_list": [sample_name], + }, + ) + initiate_pepdb_con.view.add_sample( + namespace, name, "default", "view1", ["pig_1h", "frog_0h"] + ) + assert len(initiate_pepdb_con.view.get(namespace, name, "default", "view1").samples) == 3 @pytest.mark.parametrize( "namespace, name, sample_name", @@ -1078,8 +1124,19 @@ def test_remove_sample_from_view(self, initiate_pepdb_con, namespace, name, samp ["namespace1", "amendments1", "pig_0h"], ], ) - def test_add_existing_sample_in_view(self, initiate_pepdb_con, namespace, name, sample_name): - ... + def test_remove_sample_from_view(self, initiate_pepdb_con, namespace, name, sample_name): + initiate_pepdb_con.view.create( + "view1", + { + "project_namespace": namespace, + "project_name": name, + "project_tag": "default", + "sample_list": [sample_name, "pig_1h"], + }, + ) + initiate_pepdb_con.view.remove_sample(namespace, name, "default", "view1", sample_name) + assert len(initiate_pepdb_con.view.get(namespace, name, "default", "view1").samples) == 1 + assert len(initiate_pepdb_con.project.get(namespace, name).samples) == 4 @pytest.mark.parametrize( "namespace, name, sample_name", @@ -1087,5 +1144,31 @@ def test_add_existing_sample_in_view(self, initiate_pepdb_con, namespace, name, ["namespace1", "amendments1", "pig_0h"], ], ) - def test_get_snap_view(self, initiate_pepdb_con, namespace, name, sample_name): - ... + def test_add_existing_sample_in_view(self, initiate_pepdb_con, namespace, name, sample_name): + initiate_pepdb_con.view.create( + "view1", + { + "project_namespace": namespace, + "project_name": name, + "project_tag": "default", + "sample_list": [sample_name, "pig_1h"], + }, + ) + with pytest.raises(SampleAlreadyInView): + initiate_pepdb_con.view.add_sample(namespace, name, "default", "view1", sample_name) + + @pytest.mark.parametrize( + "namespace, name, sample_name, view_name", + [ + ["namespace1", "amendments1", "pig_0h", "view1"], + ], + ) + def test_get_snap_view(self, initiate_pepdb_con, namespace, name, sample_name, view_name): + snap_project = initiate_pepdb_con.view.get_snap_view( + namespace=namespace, + name=name, + tag="default", + sample_name_list=[sample_name, "pig_1h"], + ) + + assert len(snap_project.samples) == 2 From 98a89ae0c37a4eceb85970f30063b889e3b226fa Mon Sep 17 00:00:00 2001 From: Khoroshevskyi Date: Wed, 10 Jan 2024 15:29:08 -0500 Subject: [PATCH 3/6] added get_view list function --- pepdbagent/modules/annotation.py | 25 ++++++++++++++++++++++++- tests/test_pepagent.py | 20 ++++++++++++++++++++ 2 files changed, 44 insertions(+), 1 deletion(-) diff --git a/pepdbagent/modules/annotation.py b/pepdbagent/modules/annotation.py index 99dd52f..72c79e9 100644 --- a/pepdbagent/modules/annotation.py +++ b/pepdbagent/modules/annotation.py @@ -14,7 +14,7 @@ SUBMISSION_DATE_KEY, LAST_UPDATE_DATE_KEY, ) -from pepdbagent.db_utils import BaseEngine, Projects +from pepdbagent.db_utils import BaseEngine, Projects, Views from pepdbagent.exceptions import FilterError, ProjectNotFoundError, RegistryPathError from pepdbagent.models import AnnotationList, AnnotationModel from pepdbagent.utils import convert_date_string_to_date, registry_path_converter, tuple_converter @@ -572,3 +572,26 @@ def get_by_rp_list( else: return self.get_by_rp(registry_paths, admin) + + def get_views(self, namespace: str, name: str, tag: str = DEFAULT_TAG) -> List[str]: + """ + Get list of views of the project + + :param namespace: namespace of the project + :param name: name of the project + :param tag: tag of the project + :return: list of views of the project + """ + statement = select(Views.name).where( + Views.project_mapping.has(namespace=namespace, name=name, tag=tag), + and_( + Projects.name == name, + Projects.namespace == namespace, + Projects.tag == tag, + ), + ) + views = self._pep_db_engine.session_execute(statement).all() + if views: + return [v[0] for v in views] + else: + return [] diff --git a/tests/test_pepagent.py b/tests/test_pepagent.py index 5110b62..f77d305 100644 --- a/tests/test_pepagent.py +++ b/tests/test_pepagent.py @@ -1172,3 +1172,23 @@ def test_get_snap_view(self, initiate_pepdb_con, namespace, name, sample_name, v ) assert len(snap_project.samples) == 2 + + @pytest.mark.parametrize( + "namespace, name, sample_name, view_name", + [ + ["namespace1", "amendments1", "pig_0h", "view1"], + ], + ) + def test_get_view_list_from_project( + self, initiate_pepdb_con, namespace, name, sample_name, view_name + ): + initiate_pepdb_con.view.create( + "view1", + { + "project_namespace": namespace, + "project_name": name, + "project_tag": "default", + "sample_list": [sample_name, "pig_1h"], + }, + ) + assert initiate_pepdb_con.annotation.get_views(namespace, name, "default")[0] == "view1" From 92780bcae0754e77c59d49bba00dd1becbd28da0 Mon Sep 17 00:00:00 2001 From: Khoroshevskyi Date: Thu, 11 Jan 2024 11:56:06 -0500 Subject: [PATCH 4/6] updated annotation view model --- pepdbagent/models.py | 18 +++++++++++----- pepdbagent/modules/annotation.py | 23 -------------------- pepdbagent/modules/view.py | 36 +++++++++++++++++++++++++++++++- tests/test_pepagent.py | 10 ++++++++- 4 files changed, 57 insertions(+), 30 deletions(-) diff --git a/pepdbagent/models.py b/pepdbagent/models.py index eb2116d..569fba2 100644 --- a/pepdbagent/models.py +++ b/pepdbagent/models.py @@ -164,15 +164,23 @@ class ProjectRegistryPath(BaseModel): class ViewAnnotation(BaseModel): """ - View annotation model + Project views model """ - project_namespace: str - project_name: str - project_tag: str name: str description: Optional[str] = None - number_of_samples: int + number_of_samples: int = 0 + + +class ProjectViews(BaseModel): + """ + View annotation model + """ + + namespace: str + name: str + tag: str = DEFAULT_TAG + views: List[ViewAnnotation] = [] class CreateViewDictModel(BaseModel): diff --git a/pepdbagent/modules/annotation.py b/pepdbagent/modules/annotation.py index 72c79e9..f727446 100644 --- a/pepdbagent/modules/annotation.py +++ b/pepdbagent/modules/annotation.py @@ -572,26 +572,3 @@ def get_by_rp_list( else: return self.get_by_rp(registry_paths, admin) - - def get_views(self, namespace: str, name: str, tag: str = DEFAULT_TAG) -> List[str]: - """ - Get list of views of the project - - :param namespace: namespace of the project - :param name: name of the project - :param tag: tag of the project - :return: list of views of the project - """ - statement = select(Views.name).where( - Views.project_mapping.has(namespace=namespace, name=name, tag=tag), - and_( - Projects.name == name, - Projects.namespace == namespace, - Projects.tag == tag, - ), - ) - views = self._pep_db_engine.session_execute(statement).all() - if views: - return [v[0] for v in views] - else: - return [] diff --git a/pepdbagent/modules/view.py b/pepdbagent/modules/view.py index 89c5dfa..11b07af 100644 --- a/pepdbagent/modules/view.py +++ b/pepdbagent/modules/view.py @@ -16,7 +16,7 @@ from pepdbagent.exceptions import ViewNotFoundError, SampleAlreadyInView from pepdbagent.db_utils import BaseEngine, Samples, Projects, Views, ViewSampleAssociation -from pepdbagent.models import ViewAnnotation, CreateViewDictModel +from pepdbagent.models import ViewAnnotation, CreateViewDictModel, ProjectViews _LOGGER = logging.getLogger(PKG_NAME) @@ -369,3 +369,37 @@ def get_snap_view( return peppy.Project.from_dict( {"_config": config, "_sample_dict": samples, "_subsample_dict": None} ) + + def get_views_annotation( + self, namespace: str, name: str, tag: str = DEFAULT_TAG + ) -> Union[ProjectViews, None]: + """ + Get list of views of the project + + :param namespace: namespace of the project + :param name: name of the project + :param tag: tag of the project + :return: list of views of the project + """ + statement = select(Views).where( + Views.project_mapping.has(namespace=namespace, name=name, tag=tag), + and_( + Projects.name == name, + Projects.namespace == namespace, + Projects.tag == tag, + ), + ) + views_list = [] + + with Session(self._sa_engine) as session: + views = session.scalars(statement) + for view in views: + views_list.append( + ViewAnnotation( + name=view.name, + description=view.description, + number_of_samples=len(view.samples), + ) + ) + + return ProjectViews(namespace=namespace, name=name, tag=tag, views=views_list) diff --git a/tests/test_pepagent.py b/tests/test_pepagent.py index f77d305..9cc5fa8 100644 --- a/tests/test_pepagent.py +++ b/tests/test_pepagent.py @@ -1182,6 +1182,10 @@ def test_get_snap_view(self, initiate_pepdb_con, namespace, name, sample_name, v def test_get_view_list_from_project( self, initiate_pepdb_con, namespace, name, sample_name, view_name ): + assert ( + len(initiate_pepdb_con.view.get_views_annotation(namespace, name, "default").views) + == 0 + ) initiate_pepdb_con.view.create( "view1", { @@ -1191,4 +1195,8 @@ def test_get_view_list_from_project( "sample_list": [sample_name, "pig_1h"], }, ) - assert initiate_pepdb_con.annotation.get_views(namespace, name, "default")[0] == "view1" + result = initiate_pepdb_con.view.get_views_annotation(namespace, name, "default") + assert ( + len(initiate_pepdb_con.view.get_views_annotation(namespace, name, "default").views) + == 1 + ) From d3010f7125f6d03f5dc0c9a8b38eae20647bb4e7 Mon Sep 17 00:00:00 2001 From: Khoroshevskyi Date: Thu, 11 Jan 2024 15:34:18 -0500 Subject: [PATCH 5/6] updated errors --- pepdbagent/_version.py | 2 +- pepdbagent/modules/view.py | 12 ++++++------ 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/pepdbagent/_version.py b/pepdbagent/_version.py index 9320b9c..7131288 100644 --- a/pepdbagent/_version.py +++ b/pepdbagent/_version.py @@ -1 +1 @@ -__version__ = "0.7.0a4" +__version__ = "0.7.0a5" diff --git a/pepdbagent/modules/view.py b/pepdbagent/modules/view.py index 11b07af..22517a5 100644 --- a/pepdbagent/modules/view.py +++ b/pepdbagent/modules/view.py @@ -13,7 +13,7 @@ DEFAULT_TAG, PKG_NAME, ) -from pepdbagent.exceptions import ViewNotFoundError, SampleAlreadyInView +from pepdbagent.exceptions import ViewNotFoundError, SampleAlreadyInView, ProjectNotFoundError, SampleNotFoundError from pepdbagent.db_utils import BaseEngine, Samples, Projects, Views, ViewSampleAssociation from pepdbagent.models import ViewAnnotation, CreateViewDictModel, ProjectViews @@ -158,7 +158,7 @@ def create( with Session(self._sa_engine) as sa_session: project = sa_session.scalar(project_statement) if not project: - raise ValueError( + raise ProjectNotFoundError( f"Project {view_dict.project_namespace}/{view_dict.project_name}:{view_dict.project_tag} does not exist" ) view = Views( @@ -177,7 +177,7 @@ def create( ) sample_id = sa_session.execute(sample_statement).one()[0] if not sample_id: - raise ValueError( + raise SampleNotFoundError( f"Sample {view_dict.project_namespace}/{view_dict.project_name}:{view_dict.project_tag}:{sample_name} does not exist" ) sa_session.add(ViewSampleAssociation(sample_id=sample_id, view=view)) @@ -260,7 +260,7 @@ def add_sample( ) sample = sa_session.scalar(sample_statement) if not sample: - raise ValueError( + raise SampleNotFoundError( f"Sample {namespace}/{name}:{tag}:{sample_name} does not exist" ) try: @@ -346,7 +346,7 @@ def get_snap_view( with Session(self._sa_engine) as sa_session: project = sa_session.scalar(project_statement) if not project: - raise ValueError(f"Project {namespace}/{name}:{tag} does not exist") + raise ProjectNotFoundError(f"Project {namespace}/{name}:{tag} does not exist") samples = [] for sample_name in sample_name_list: sample_statement = select(Samples).where( @@ -357,7 +357,7 @@ def get_snap_view( ) sample = sa_session.scalar(sample_statement) if not sample: - raise ValueError( + raise SampleNotFoundError( f"Sample {namespace}/{name}:{tag}:{sample_name} does not exist" ) samples.append(sample.sample) From 9105e9a10f64b53735eb444933cd19521e83faeb Mon Sep 17 00:00:00 2001 From: Khoroshevskyi Date: Thu, 11 Jan 2024 18:53:08 -0500 Subject: [PATCH 6/6] Improvement of updating projects --- pepdbagent/modules/annotation.py | 2 +- pepdbagent/modules/project.py | 101 +++++++++++++++++++++++++++---- pepdbagent/modules/view.py | 12 ++-- tests/test_pepagent.py | 7 +-- 4 files changed, 98 insertions(+), 24 deletions(-) diff --git a/pepdbagent/modules/annotation.py b/pepdbagent/modules/annotation.py index f727446..99dd52f 100644 --- a/pepdbagent/modules/annotation.py +++ b/pepdbagent/modules/annotation.py @@ -14,7 +14,7 @@ SUBMISSION_DATE_KEY, LAST_UPDATE_DATE_KEY, ) -from pepdbagent.db_utils import BaseEngine, Projects, Views +from pepdbagent.db_utils import BaseEngine, Projects from pepdbagent.exceptions import FilterError, ProjectNotFoundError, RegistryPathError from pepdbagent.models import AnnotationList, AnnotationModel from pepdbagent.utils import convert_date_string_to_date, registry_path_converter, tuple_converter diff --git a/pepdbagent/modules/project.py b/pepdbagent/modules/project.py index c8e5f55..78dc953 100644 --- a/pepdbagent/modules/project.py +++ b/pepdbagent/modules/project.py @@ -1,7 +1,7 @@ import datetime import json import logging -from typing import Union, List, NoReturn +from typing import Union, List, NoReturn, Mapping import peppy from sqlalchemy import and_, delete, select @@ -90,11 +90,16 @@ def get( subsample_list = list(subsample_dict.values()) else: subsample_list = [] + + # samples + samples_dict = { + sample_sa.row_number: sample_sa.sample + for sample_sa in found_prj.samples_mapping + } + project_value = { CONFIG_KEY: found_prj.config, - SAMPLE_RAW_DICT_KEY: [ - sample_sa.sample for sample_sa in found_prj.samples_mapping - ], + SAMPLE_RAW_DICT_KEY: [samples_dict[key] for key in sorted(samples_dict)], SUBSAMPLE_RAW_LIST_KEY: subsample_list, } # project_value = found_prj.project_value @@ -466,16 +471,25 @@ def update( found_prj.name = found_prj.config[NAME_KEY] if "samples" in update_dict: - if found_prj.samples_mapping: - for sample in found_prj.samples_mapping: - _LOGGER.debug(f"deleting samples: {str(sample)}") - session.delete(sample) - - self._add_samples_to_project( - found_prj, - update_dict["samples"], - sample_table_index=update_dict["config"].get(SAMPLE_TABLE_INDEX_KEY), + self._update_samples( + namespace=namespace, + name=name, + tag=tag, + samples_list=update_dict["samples"], + sample_name_key=update_dict["config"].get( + SAMPLE_TABLE_INDEX_KEY, "sample_name" + ), ) + # if found_prj.samples_mapping: + # for sample in found_prj.samples_mapping: + # _LOGGER.debug(f"deleting samples: {str(sample)}") + # session.delete(sample) + # + # self._add_samples_to_project( + # found_prj, + # update_dict["samples"], + # sample_table_index=update_dict["config"].get(SAMPLE_TABLE_INDEX_KEY), + # ) if "subsamples" in update_dict: if found_prj.subsamples_mapping: @@ -496,6 +510,67 @@ def update( else: raise ProjectNotFoundError("No items will be updated!") + def _update_samples( + self, + namespace: str, + name: str, + tag: str, + samples_list: List[Mapping], + sample_name_key: str = "sample_name", + ) -> None: + """ + Update samples in the project + This is a new method that instead of deleting all samples and adding new ones, + updates samples and adds new ones if they don't exist + + :param samples_list: list of samples to be updated + :param sample_name_key: key of the sample name + :return: None + """ + new_sample_names = [sample[sample_name_key] for sample in samples_list] + with Session(self._sa_engine) as session: + project = session.scalar( + select(Projects).where( + and_( + Projects.namespace == namespace, Projects.name == name, Projects.tag == tag + ) + ) + ) + old_sample_names = [sample.sample_name for sample in project.samples_mapping] + for old_sample in old_sample_names: + if old_sample not in new_sample_names: + session.execute( + delete(Samples).where( + and_( + Samples.sample_name == old_sample, Samples.project_id == project.id + ) + ) + ) + + order_number = 0 + for new_sample in samples_list: + order_number += 1 + if new_sample[sample_name_key] not in old_sample_names: + project.samples_mapping.append( + Samples( + sample=new_sample, + sample_name=new_sample[sample_name_key], + row_number=order_number, + ) + ) + else: + sample_mapping = session.scalar( + select(Samples).where( + and_( + Samples.sample_name == new_sample[sample_name_key], + Samples.project_id == project.id, + ) + ) + ) + sample_mapping.sample = new_sample + sample_mapping.row_number = order_number + session.commit() + @staticmethod def __create_update_dict(update_values: UpdateItems) -> dict: """ diff --git a/pepdbagent/modules/view.py b/pepdbagent/modules/view.py index 22517a5..fe41be6 100644 --- a/pepdbagent/modules/view.py +++ b/pepdbagent/modules/view.py @@ -13,7 +13,12 @@ DEFAULT_TAG, PKG_NAME, ) -from pepdbagent.exceptions import ViewNotFoundError, SampleAlreadyInView, ProjectNotFoundError, SampleNotFoundError +from pepdbagent.exceptions import ( + ViewNotFoundError, + SampleAlreadyInView, + ProjectNotFoundError, + SampleNotFoundError, +) from pepdbagent.db_utils import BaseEngine, Samples, Projects, Views, ViewSampleAssociation from pepdbagent.models import ViewAnnotation, CreateViewDictModel, ProjectViews @@ -383,11 +388,6 @@ def get_views_annotation( """ statement = select(Views).where( Views.project_mapping.has(namespace=namespace, name=name, tag=tag), - and_( - Projects.name == name, - Projects.namespace == namespace, - Projects.tag == tag, - ), ) views_list = [] diff --git a/tests/test_pepagent.py b/tests/test_pepagent.py index 9cc5fa8..91b53de 100644 --- a/tests/test_pepagent.py +++ b/tests/test_pepagent.py @@ -324,9 +324,9 @@ def test_update_project_description( "namespace, name", [ ["namespace1", "amendments1"], - ["namespace1", "amendments2"], - ["namespace2", "derive"], - ["namespace2", "imply"], + # ["namespace1", "amendments2"], + # ["namespace2", "derive"], + # ["namespace2", "imply"], ], ) def test_update_whole_project(self, initiate_pepdb_con, namespace, name): @@ -1195,7 +1195,6 @@ def test_get_view_list_from_project( "sample_list": [sample_name, "pig_1h"], }, ) - result = initiate_pepdb_con.view.get_views_annotation(namespace, name, "default") assert ( len(initiate_pepdb_con.view.get_views_annotation(namespace, name, "default").views) == 1