From 8c560a9f6ae11ab8f45ca9f37395d0fa7e259bb0 Mon Sep 17 00:00:00 2001 From: Khoroshevskyi Date: Fri, 19 Jan 2024 16:34:31 -0500 Subject: [PATCH 1/5] Fixed stared project annotation --- pepdbagent/modules/user.py | 29 +++++++++++++++-------------- 1 file changed, 15 insertions(+), 14 deletions(-) diff --git a/pepdbagent/modules/user.py b/pepdbagent/modules/user.py index 5670195..5b27577 100644 --- a/pepdbagent/modules/user.py +++ b/pepdbagent/modules/user.py @@ -166,22 +166,23 @@ def get_favorites(self, namespace: str) -> AnnotationList: number_of_projects = len([kk.project_mapping for kk in query_result.stars_mapping]) project_list = [] for prj_list in query_result.stars_mapping: + prj = prj_list.project_mapping project_list.append( AnnotationModel( - namespace=prj_list.project_mapping.namespace, - name=prj_list.project_mapping.name, - tag=prj_list.project_mapping.tag, - is_private=prj_list.project_mapping.private, - number_of_samples=prj_list.project_mapping.number_of_samples, - description=prj_list.project_mapping.description, - last_update_date=str(prj_list.project_mapping.last_update_date), - submission_date=str(prj_list.project_mapping.submission_date), - digest=prj_list.project_mapping.digest, - pep_schema=prj_list.project_mapping.pep_schema, - pop=prj_list.project_mapping.pop, - stars_number=prj_list.project_mapping.number_of_stars, - forked_from=f"{prj_list.project_mapping.namespace}/{prj_list.project_mapping.name}:{prj_list.project_mapping.tag}" - if prj_list.project_mapping + namespace=prj.namespace, + name=prj.name, + tag=prj.tag, + is_private=prj.private, + number_of_samples=prj.number_of_samples, + description=prj.description, + last_update_date=str(prj.last_update_date), + submission_date=str(prj.submission_date), + digest=prj.digest, + pep_schema=prj.pep_schema, + pop=prj.pop, + stars_number=prj.number_of_stars, + forked_from=f"{prj.forked_from_mapping.namespace}/{prj.forked_from_mapping.name}:{prj.forked_from_mapping.tag}" + if prj.forked_from_mapping else None, ) ) From 1b9ea7bdce8066722f2f7753b3912658ce2a0be2 Mon Sep 17 00:00:00 2001 From: Khoroshevskyi Date: Fri, 19 Jan 2024 16:34:31 -0500 Subject: [PATCH 2/5] Fixed stared project annotation --- docs/changelog.md | 4 ++++ pepdbagent/_version.py | 2 +- pepdbagent/modules/user.py | 29 +++++++++++++++-------------- 3 files changed, 20 insertions(+), 15 deletions(-) diff --git a/docs/changelog.md b/docs/changelog.md index b7d5430..8f3e755 100644 --- a/docs/changelog.md +++ b/docs/changelog.md @@ -2,6 +2,10 @@ This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html) and [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) format. +## [0.7.1] -- 2023-01-19 +- Fixed bug in Stars annotation + + ## [0.7.0] -- 2023-01-17 - Added `pop` to project table and annotation model [#107](https://github.com/pepkit/pepdbagent/issues/107) - Added `forked_from` feature [#73](https://github.com/pepkit/pepdbagent/issues/73) diff --git a/pepdbagent/_version.py b/pepdbagent/_version.py index 49e0fc1..a5f830a 100644 --- a/pepdbagent/_version.py +++ b/pepdbagent/_version.py @@ -1 +1 @@ -__version__ = "0.7.0" +__version__ = "0.7.1" diff --git a/pepdbagent/modules/user.py b/pepdbagent/modules/user.py index 5670195..5b27577 100644 --- a/pepdbagent/modules/user.py +++ b/pepdbagent/modules/user.py @@ -166,22 +166,23 @@ def get_favorites(self, namespace: str) -> AnnotationList: number_of_projects = len([kk.project_mapping for kk in query_result.stars_mapping]) project_list = [] for prj_list in query_result.stars_mapping: + prj = prj_list.project_mapping project_list.append( AnnotationModel( - namespace=prj_list.project_mapping.namespace, - name=prj_list.project_mapping.name, - tag=prj_list.project_mapping.tag, - is_private=prj_list.project_mapping.private, - number_of_samples=prj_list.project_mapping.number_of_samples, - description=prj_list.project_mapping.description, - last_update_date=str(prj_list.project_mapping.last_update_date), - submission_date=str(prj_list.project_mapping.submission_date), - digest=prj_list.project_mapping.digest, - pep_schema=prj_list.project_mapping.pep_schema, - pop=prj_list.project_mapping.pop, - stars_number=prj_list.project_mapping.number_of_stars, - forked_from=f"{prj_list.project_mapping.namespace}/{prj_list.project_mapping.name}:{prj_list.project_mapping.tag}" - if prj_list.project_mapping + namespace=prj.namespace, + name=prj.name, + tag=prj.tag, + is_private=prj.private, + number_of_samples=prj.number_of_samples, + description=prj.description, + last_update_date=str(prj.last_update_date), + submission_date=str(prj.submission_date), + digest=prj.digest, + pep_schema=prj.pep_schema, + pop=prj.pop, + stars_number=prj.number_of_stars, + forked_from=f"{prj.forked_from_mapping.namespace}/{prj.forked_from_mapping.name}:{prj.forked_from_mapping.tag}" + if prj.forked_from_mapping else None, ) ) From 6396001c626cfeeeacefcc47b164b8c04acd979a Mon Sep 17 00:00:00 2001 From: Khoroshevskyi Date: Mon, 22 Jan 2024 12:21:29 -0500 Subject: [PATCH 3/5] Improved query of annotation retrival --- docs/changelog.md | 4 +- pepdbagent/models.py | 6 +++ pepdbagent/modules/annotation.py | 78 +++++++++++++++++++++++++++++--- pepdbagent/modules/user.py | 3 +- pepdbagent/modules/view.py | 14 ++++++ tests/test_pepagent.py | 15 ++++++ 6 files changed, 109 insertions(+), 11 deletions(-) diff --git a/docs/changelog.md b/docs/changelog.md index 8f3e755..3ce88f1 100644 --- a/docs/changelog.md +++ b/docs/changelog.md @@ -2,9 +2,9 @@ This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html) and [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) format. -## [0.7.1] -- 2023-01-19 +## [0.7.1] -- 2023-01-22 - Fixed bug in Stars annotation - +- SQL efficiency improvements ## [0.7.0] -- 2023-01-17 - Added `pop` to project table and annotation model [#107](https://github.com/pepkit/pepdbagent/issues/107) diff --git a/pepdbagent/models.py b/pepdbagent/models.py index a7f08b9..6c5c4a6 100644 --- a/pepdbagent/models.py +++ b/pepdbagent/models.py @@ -194,3 +194,9 @@ class CreateViewDictModel(BaseModel): project_name: str project_tag: str sample_list: List[str] + + +class RegistryPath(BaseModel): + namespace: str + name: str + tag: Optional[str] = "default" diff --git a/pepdbagent/modules/annotation.py b/pepdbagent/modules/annotation.py index 0876840..b7bbcf5 100644 --- a/pepdbagent/modules/annotation.py +++ b/pepdbagent/modules/annotation.py @@ -16,7 +16,7 @@ ) from pepdbagent.db_utils import BaseEngine, Projects from pepdbagent.exceptions import FilterError, ProjectNotFoundError, RegistryPathError -from pepdbagent.models import AnnotationList, AnnotationModel +from pepdbagent.models import AnnotationList, AnnotationModel, RegistryPath from pepdbagent.utils import convert_date_string_to_date, registry_path_converter, tuple_converter _LOGGER = logging.getLogger(PKG_NAME) @@ -309,7 +309,7 @@ def _get_projects( if admin is None: admin = [] - statement = select(Projects.id) + statement = select(Projects) statement = self._add_condition( statement, @@ -325,13 +325,10 @@ def _get_projects( if pep_type: statement = statement.where(Projects.pop.is_(pep_type == "pop")) - id_results = self._pep_db_engine.session_execute(statement).all() - results_list = [] with Session(self._sa_engine) as session: - for prj_ids in id_results: - result = session.scalar(select(Projects).where(Projects.id == prj_ids[0])) - + results = session.scalars(statement) + for result in results: results_list.append( AnnotationModel( namespace=result.namespace, @@ -572,3 +569,70 @@ def get_by_rp_list( else: return self.get_by_rp(registry_paths, admin) + + def get_projects_list( + self, + namespace: str = None, + search_str: str = None, + admin: Union[str, List[str]] = None, + limit: int = DEFAULT_LIMIT, + offset: int = DEFAULT_OFFSET, + order_by: str = "update_date", + order_desc: bool = False, + filter_by: Optional[Literal["submission_date", "last_update_date"]] = None, + filter_start_date: Optional[str] = None, + filter_end_date: Optional[str] = None, + pep_type: Optional[Literal["pep", "pop"]] = None, + ) -> List[RegistryPath]: + """ + Retrieve a list of projects by providing a search string. + This function serves as a lightweight version of the full 'get' function, + returning only a list of registry paths without annotations. + It is designed for use cases where a large list of projects is needed with minimal processing time. + + :param namespace: namespace where to search for a project + :param search_str: search string that has to be found in the name or tag + :param admin: True, if user is admin of the namespace [Default: False] + :param limit: limit of return results + :param offset: number of results off set (that were already showed) + :param order_by: sort the result-set by the information + Options: ["name", "update_date", "submission_date"] + [Default: "update_date"] + :param order_desc: Sort the records in descending order. [Default: False] + :param filter_by: data to use filter on. + Options: ["submission_date", "last_update_date"] + [Default: filter won't be used] + :param filter_start_date: Filter start date. Format: "YYYY:MM:DD" + :param filter_end_date: Filter end date. Format: "YYYY:MM:DD". if None: present date will be used + :param pep_type: Get pep with specified type. Options: ["pep", "pop"]. Default: None, get all peps + :return: list of found projects with their annotations. + """ + _LOGGER.info(f"Running project search: (namespace: {namespace}, query: {search_str}.") + + if admin is None: + admin = [] + statement = select(Projects.namespace, Projects.name, Projects.tag) + + statement = self._add_condition( + statement, + namespace=namespace, + search_str=search_str, + admin_list=admin, + ) + statement = self._add_date_filter_if_provided( + statement, filter_by, filter_start_date, filter_end_date + ) + statement = self._add_order_by_keyword(statement, by=order_by, desc=order_desc) + statement = statement.limit(limit).offset(offset) + if pep_type: + statement = statement.where(Projects.pop.is_(pep_type == "pop")) + + results_list = [] + with Session(self._sa_engine) as session: + results = session.execute(statement) + + for result in results: + results_list.append( + RegistryPath(namespace=result[0], name=result[1], tag=result[2]) + ) + return results_list diff --git a/pepdbagent/modules/user.py b/pepdbagent/modules/user.py index 5b27577..dbd6c92 100644 --- a/pepdbagent/modules/user.py +++ b/pepdbagent/modules/user.py @@ -10,7 +10,6 @@ ) from pepdbagent.db_utils import BaseEngine, User, Stars, Projects -from pepdbagent.modules.project import PEPDatabaseProject from pepdbagent.models import AnnotationList, AnnotationModel from pepdbagent.exceptions import ProjectNotInFavorites, ProjectAlreadyInFavorites @@ -114,7 +113,7 @@ def remove_project_from_favorites( :return: None """ _LOGGER.debug( - f"Removing project {project_namespace}/{project_name}:{project_tag} from fProjectNotInFavorites for user {namespace}" + f"Removing project {project_namespace}/{project_name}:{project_tag} from favorites in {namespace}" ) user_id = self.get_user_id(namespace) diff --git a/pepdbagent/modules/view.py b/pepdbagent/modules/view.py index fe41be6..aa43f86 100644 --- a/pepdbagent/modules/view.py +++ b/pepdbagent/modules/view.py @@ -67,6 +67,7 @@ def get( _subsample_dict: dict } """ + _LOGGER.debug(f"Get view {view_name} from {namespace}/{name}:{tag}") view_statement = select(Views).where( and_( Views.project_mapping.has(namespace=namespace, name=name, tag=tag), @@ -106,6 +107,7 @@ def get_annotation( description: str, number_of_samples: int} """ + _LOGGER.debug(f"Get annotation for view {view_name} in {namespace}/{name}:{tag}") view_statement = select(Views).where( and_( Views.project_mapping.has(namespace=namespace, name=name, tag=tag), @@ -149,6 +151,7 @@ def create( :param description: description of the view retrun: None """ + _LOGGER.debug(f"Creating view {view_name} with provided info: (view_dict: {view_dict})") if isinstance(view_dict, dict): view_dict = CreateViewDictModel(**view_dict) @@ -205,6 +208,9 @@ def delete( :param view_name: name of the view :return: None """ + _LOGGER.debug( + f"Deleting view {view_name} from {project_namespace}/{project_name}:{project_tag}" + ) view_statement = select(Views).where( and_( Views.project_mapping.has( @@ -241,6 +247,9 @@ def add_sample( :param sample_name: sample name :return: None """ + _LOGGER.debug( + f"Adding sample {sample_name} to view {view_name} in {namespace}/{name}:{tag}" + ) if isinstance(sample_name, str): sample_name = [sample_name] view_statement = select(Views).where( @@ -296,6 +305,9 @@ def remove_sample( :param sample_name: sample name :return: None """ + _LOGGER.debug( + f"Removing sample {sample_name} from view {view_name} in {namespace}/{name}:{tag}" + ) view_statement = select(Views).where( and_( Views.project_mapping.has(namespace=namespace, name=name, tag=tag), @@ -341,6 +353,7 @@ def get_snap_view( :param raw: retrieve unprocessed (raw) PEP dict. :return: peppy.Project object """ + _LOGGER.debug(f"Creating snap view for {namespace}/{name}:{tag}") project_statement = select(Projects).where( and_( Projects.namespace == namespace, @@ -386,6 +399,7 @@ def get_views_annotation( :param tag: tag of the project :return: list of views of the project """ + _LOGGER.debug(f"Get views annotation for {namespace}/{name}:{tag}") statement = select(Views).where( Views.project_mapping.has(namespace=namespace, name=name, tag=tag), ) diff --git a/tests/test_pepagent.py b/tests/test_pepagent.py index 91b53de..edc7979 100644 --- a/tests/test_pepagent.py +++ b/tests/test_pepagent.py @@ -679,6 +679,21 @@ def test_search_incorrect_incorrect_pep_type( with pytest.raises(ValueError): initiate_pepdb_con.annotation.get(namespace=namespace, pep_type="incorrect") + @pytest.mark.parametrize( + "namespace, query, found_number", + [ + ["namespace1", "ame", 2], + ], + ) + def test_project_list_without_annotation( + self, initiate_pepdb_con, namespace, query, found_number + ): + result = initiate_pepdb_con.annotation.get_projects_list( + namespace=namespace, + search_str=query, + ) + assert len(result) == found_number + @pytest.mark.skipif( not db_setup(), From 57963e6bcbc37b2caa20134f48aed38f86a690ee Mon Sep 17 00:00:00 2001 From: Khoroshevskyi Date: Mon, 22 Jan 2024 16:39:57 -0500 Subject: [PATCH 4/5] Added sorting of star projects --- docs/changelog.md | 1 + pepdbagent/db_utils.py | 9 +++++++-- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/docs/changelog.md b/docs/changelog.md index 3ce88f1..3bc3396 100644 --- a/docs/changelog.md +++ b/docs/changelog.md @@ -5,6 +5,7 @@ This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.htm ## [0.7.1] -- 2023-01-22 - Fixed bug in Stars annotation - SQL efficiency improvements +- Added sort by date in stared projects ## [0.7.0] -- 2023-01-17 - Added `pop` to project table and annotation model [#107](https://github.com/pepkit/pepdbagent/issues/107) diff --git a/pepdbagent/db_utils.py b/pepdbagent/db_utils.py index b48fb3f..33e92e8 100644 --- a/pepdbagent/db_utils.py +++ b/pepdbagent/db_utils.py @@ -88,7 +88,7 @@ class Projects(Base): number_of_stars: Mapped[int] = mapped_column(default=0) submission_date: Mapped[datetime.datetime] last_update_date: Mapped[Optional[datetime.datetime]] = mapped_column( - onupdate=deliver_update_date, default=deliver_update_date + default=deliver_update_date, # onupdate=deliver_update_date, # This field should not be updated, while we are adding project to favorites ) pep_schema: Mapped[Optional[str]] pop: Mapped[Optional[bool]] = mapped_column(default=False) @@ -168,7 +168,9 @@ class User(Base): id: Mapped[int] = mapped_column(primary_key=True) namespace: Mapped[str] stars_mapping: Mapped[List["Stars"]] = relationship( - back_populates="user_mapping", cascade="all, delete-orphan" + back_populates="user_mapping", + cascade="all, delete-orphan", + order_by="Stars.star_date.desc()", ) @@ -183,6 +185,9 @@ class Stars(Base): project_id = mapped_column(ForeignKey("projects.id", ondelete="CASCADE"), primary_key=True) user_mapping: Mapped[List["User"]] = relationship(back_populates="stars_mapping") project_mapping: Mapped["Projects"] = relationship(back_populates="stars_mapping") + star_date: Mapped[datetime.datetime] = mapped_column( + onupdate=deliver_update_date, default=deliver_update_date + ) class Views(Base): From 5a0ba8ff5ab8786832c0a95d96bac705c3f424ab Mon Sep 17 00:00:00 2001 From: Khoroshevskyi Date: Mon, 22 Jan 2024 17:32:00 -0500 Subject: [PATCH 5/5] Updated project retrival statement --- pepdbagent/modules/project.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pepdbagent/modules/project.py b/pepdbagent/modules/project.py index 8c89b54..6aee03b 100644 --- a/pepdbagent/modules/project.py +++ b/pepdbagent/modules/project.py @@ -75,7 +75,7 @@ def get( try: with Session(self._sa_engine) as session: - found_prj = session.scalars(statement).one() + found_prj = session.scalar(statement) if found_prj: _LOGGER.info(