Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Release 0.11.1 #147

Merged
merged 12 commits into from
Sep 18, 2024
4 changes: 4 additions & 0 deletions docs/changelog.md
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,10 @@

This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html) and [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) format.

## [0.11.1] -- 2024-09-04
- Added archive table of namespaces
- Added sort by stars

## [0.11.0] -- 2024-07-24
- Added validation schemas

Expand Down
2 changes: 1 addition & 1 deletion pepdbagent/_version.py
Original file line number Diff line number Diff line change
@@ -1 +1 @@
__version__ = "0.11.0"
__version__ = "0.11.1"
24 changes: 24 additions & 0 deletions pepdbagent/db_utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -364,6 +364,30 @@ class SchemaGroupRelations(Base):
)


class TarNamespace(Base):

__tablename__ = "namespace_archives"

id: Mapped[int] = mapped_column(primary_key=True)
namespace: Mapped[str] = mapped_column(ForeignKey("users.namespace", ondelete="CASCADE"))
file_path: Mapped[str] = mapped_column(nullable=False)
creation_date: Mapped[datetime.datetime] = mapped_column(default=deliver_update_date)
number_of_projects: Mapped[int] = mapped_column(default=0)
file_size: Mapped[int] = mapped_column(nullable=False)


class BedBaseStats(Base):
__tablename__ = "bedbase_stats"

id: Mapped[int] = mapped_column(primary_key=True)
gse: Mapped[str] = mapped_column()
gsm: Mapped[str] = mapped_column()
sample_name: Mapped[str] = mapped_column(nullable=True)
genome: Mapped[Optional[str]] = mapped_column(nullable=True, default="")
last_update_date: Mapped[Optional[str]] = mapped_column()
submission_date: Mapped[Optional[str]] = mapped_column()


class BaseEngine:
"""
A class with base methods, that are used in several classes. e.g. fetch_one or fetch_all
Expand Down
24 changes: 23 additions & 1 deletion pepdbagent/models.py
Original file line number Diff line number Diff line change
Expand Up @@ -280,7 +280,7 @@ class SchemaGroupAnnotation(BaseModel):

namespace: str
name: str
description: Optional[str]
description: Optional[str] = ""
schemas: List[SchemaAnnotation]


Expand All @@ -293,3 +293,25 @@ class SchemaGroupSearchResult(BaseModel):
limit: int
offset: int
results: List[SchemaGroupAnnotation]


class TarNamespaceModel(BaseModel):
"""
Namespace archive model
"""

identifier: int = None
namespace: str
file_path: str
creation_date: datetime.datetime = None
number_of_projects: int = 0
file_size: int = 0


class TarNamespaceModelReturn(BaseModel):
"""
Namespace archive search model
"""

count: int
results: List[TarNamespaceModel]
8 changes: 5 additions & 3 deletions pepdbagent/modules/annotation.py
Original file line number Diff line number Diff line change
Expand Up @@ -300,7 +300,7 @@ def _get_projects(
:param limit: limit of return results
:param offset: number of results off set (that were already showed)
:param order_by: sort the result-set by the information
Options: ["name", "update_date", "submission_date"]
Options: ["update_date", "name", "submission_date", "stars"]
[Default: "update_date"]
:param order_desc: Sort the records in descending order. [Default: False]
:param filter_by: data to use filter on.
Expand Down Expand Up @@ -371,7 +371,7 @@ def _add_order_by_keyword(

:param statement: sqlalchemy representation of a SELECT statement.
:param by: sort the result-set by the information
Options: ["name", "update_date", "submission_date"]
Options: ["name", "update_date", "submission_date", "stars"]
[Default: "update_date"]
:param desc: Sort the records in descending order. [Default: False]
:return: sqlalchemy representation of a SELECT statement with order by keyword
Expand All @@ -382,6 +382,8 @@ def _add_order_by_keyword(
order_by_obj = Projects.name
elif by == SUBMISSION_DATE_KEY:
order_by_obj = Projects.submission_date
elif by == "stars":
order_by_obj = Projects.number_of_stars
else:
_LOGGER.warning(
f"order by: '{by}' statement is unavailable. Projects are sorted by 'update_date'"
Expand Down Expand Up @@ -614,7 +616,7 @@ def get_projects_list(
:param limit: limit of return results
:param offset: number of results off set (that were already showed)
:param order_by: sort the result-set by the information
Options: ["name", "update_date", "submission_date"]
Options: ["name", "update_date", "submission_date", "stars"]
[Default: "update_date"]
:param order_desc: Sort the records in descending order. [Default: False]
:param filter_by: data to use filter on.
Expand Down
76 changes: 74 additions & 2 deletions pepdbagent/modules/namespace.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,19 +3,21 @@
from datetime import datetime, timedelta
from typing import List, Tuple, Union

from sqlalchemy import distinct, func, or_, select
from sqlalchemy import distinct, func, or_, select, delete
from sqlalchemy.orm import Session
from sqlalchemy.sql.selectable import Select

from pepdbagent.const import DEFAULT_LIMIT, DEFAULT_LIMIT_INFO, DEFAULT_OFFSET, PKG_NAME
from pepdbagent.db_utils import BaseEngine, Projects, User
from pepdbagent.db_utils import BaseEngine, Projects, User, TarNamespace
from pepdbagent.exceptions import NamespaceNotFoundError
from pepdbagent.models import (
ListOfNamespaceInfo,
Namespace,
NamespaceInfo,
NamespaceList,
NamespaceStats,
TarNamespaceModel,
TarNamespaceModelReturn,
)
from pepdbagent.utils import tuple_converter

Expand Down Expand Up @@ -300,3 +302,73 @@ def stats(self, namespace: str = None, monthly: bool = False) -> NamespaceStats:
projects_updated=counts_last_update,
projects_created=counts_submission,
)

def upload_tar_info(self, tar_info: TarNamespaceModel) -> None:
"""
Upload metadata of tar GEO files

tar_info: TarNamespaceModel
:return: None
"""

with Session(self._sa_engine) as session:
new_tar = TarNamespace(
file_path=tar_info.file_path,
namespace=tar_info.namespace,
creation_date=tar_info.creation_date,
number_of_projects=tar_info.number_of_projects,
file_size=tar_info.file_size,
)
session.add(new_tar)
session.commit()

_LOGGER.info("Geo tar info was uploaded successfully!")

def get_tar_info(self, namespace: str) -> TarNamespaceModelReturn:
"""
Get metadata of tar GEO files

:param namespace: namespace of the tar files

:return: list with geo data
"""

with Session(self._sa_engine) as session:
tar_info = session.scalars(
select(TarNamespace)
.where(TarNamespace.namespace == namespace)
.order_by(TarNamespace.creation_date.desc())
)

results = []
for result in tar_info:
results.append(
TarNamespaceModel(
identifier=result.id,
namespace=result.namespace,
file_path=result.file_path,
creation_date=result.creation_date,
number_of_projects=result.number_of_projects,
file_size=result.file_size,
)
)

return TarNamespaceModelReturn(count=len(results), results=results)

def delete_tar_info(self, namespace: str = None) -> None:
"""
Delete all metadata of tar GEO files

:param namespace: namespace of the tar files

:return: None
"""

with Session(self._sa_engine) as session:

delete_statement = delete(TarNamespace)
if namespace:
delete_statement = delete_statement.where(TarNamespace.namespace == namespace)
session.execute(delete_statement)
khoroshevskyi marked this conversation as resolved.
Show resolved Hide resolved
session.commit()
_LOGGER.info("Geo tar info was deleted successfully!")
6 changes: 5 additions & 1 deletion pepdbagent/modules/project.py
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,7 @@
Samples,
Schemas,
Subsamples,
TarNamespace,
UpdateTypes,
User,
)
Expand All @@ -46,6 +47,8 @@
SchemaDoesNotExistError,
)
from pepdbagent.models import (
TarNamespaceModel,
TarNamespaceModelReturn,
HistoryAnnotationModel,
HistoryChangeModel,
ProjectDict,
Expand Down Expand Up @@ -657,6 +660,7 @@ def update(
),
history_sa_model=new_history,
)
found_prj.number_of_samples = len(update_dict["samples"])

if "subsamples" in update_dict:
if found_prj.subsamples_mapping:
Expand Down Expand Up @@ -1412,7 +1416,7 @@ def restore(

def clean_history(self, days: int = 90) -> None:
"""
Delete all history data that is older then 3 month, or specific number of days
Delete all history data that is older than 3 month, or specific number of days

:param days: number of days to keep history data
:return: None
Expand Down
2 changes: 1 addition & 1 deletion requirements/requirements-all.txt
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
sqlalchemy>=2.0.0
logmuse>=0.2.7
peppy>=0.40.4
peppy>=0.40.6
ubiquerg>=0.6.2
coloredlogs>=15.0.1
pytest-mock
Expand Down
49 changes: 49 additions & 0 deletions tests/test_tar_meta.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,49 @@
from datetime import datetime

import pytest

from pepdbagent.models import TarNamespaceModel

from .utils import PEPDBAgentContextManager


@pytest.mark.skipif(
not PEPDBAgentContextManager().db_setup(),
reason="DB is not setup",
)
class TestGeoTar:
"""
Test project methods
"""

test_namespace = "namespace1"

tar_info = TarNamespaceModel(
namespace=test_namespace,
submission_date=datetime.now(),
start_period=datetime.now(),
end_period=datetime.now(),
number_of_projects=1,
file_path="blabla/test.tar",
)

def test_create_meta_tar(self):
with PEPDBAgentContextManager(add_data=True) as agent:

agent.namespace.upload_tar_info(tar_info=self.tar_info)

result = agent.namespace.get_tar_info(namespace=self.test_namespace)

assert result.count == 1

def test_delete_meta_tar(self):
with PEPDBAgentContextManager(add_data=True) as agent:
agent.namespace.upload_tar_info(tar_info=self.tar_info)

result = agent.namespace.get_tar_info(namespace=self.test_namespace)
assert result.count == 1

agent.namespace.delete_tar_info()

result = agent.namespace.get_tar_info(namespace=self.test_namespace)
assert result.count == 0
Loading