diff --git a/.env.docker.example b/.env.docker.example index 6ab156d90..e27476a47 100644 --- a/.env.docker.example +++ b/.env.docker.example @@ -8,5 +8,4 @@ MARIADB_TEST_DATABASE=uvlhubdb_test MARIADB_USER=uvlhubdb_user MARIADB_PASSWORD=uvlhubdb_password MARIADB_ROOT_PASSWORD=uvlhubdb_root_password -WEBHOOK_TOKEN=asdfasdfasda WORKING_DIR=/app/ diff --git a/.flake8 b/.flake8 index 79a16af7e..6deafc261 100644 --- a/.flake8 +++ b/.flake8 @@ -1,2 +1,2 @@ [flake8] -max-line-length = 120 \ No newline at end of file +max-line-length = 120 diff --git a/.gitignore b/.gitignore index b16069006..71b757e79 100644 --- a/.gitignore +++ b/.gitignore @@ -15,4 +15,5 @@ ubuntu-bionic-18.04-cloudimg-console.log nginx.prod.ssl.conf .version entrypoint.sh -deployments.log \ No newline at end of file +deployments.log +test_file.txt \ No newline at end of file diff --git a/app/__init__.py b/app/__init__.py index 36de9ab2a..f13bfbcfa 100644 --- a/app/__init__.py +++ b/app/__init__.py @@ -94,16 +94,4 @@ def get_authenticated_user(): return None -def datasets_counter() -> int: - from app.blueprints.dataset.models import DataSet - count = DataSet.query.count() - return count - - -def feature_models_counter() -> int: - from app.blueprints.dataset.models import FeatureModel - count = FeatureModel.query.count() - return count - - app = create_app() diff --git a/app/blueprints/auth/models.py b/app/blueprints/auth/models.py index b70f14b3d..dec91591d 100644 --- a/app/blueprints/auth/models.py +++ b/app/blueprints/auth/models.py @@ -1,4 +1,4 @@ -from datetime import datetime +from datetime import datetime, timezone from flask_login import UserMixin from werkzeug.security import generate_password_hash, check_password_hash @@ -11,7 +11,7 @@ class User(db.Model, UserMixin): email = db.Column(db.String(256), unique=True, nullable=False) password = db.Column(db.String(256), nullable=False) - created_at = db.Column(db.DateTime, nullable=False, default=datetime.utcnow) + created_at = db.Column(db.DateTime, nullable=False, default=lambda: datetime.now(timezone.utc)) data_sets = db.relationship('DataSet', backref='user', lazy=True) profile = db.relationship('UserProfile', backref='user', uselist=False) @@ -38,15 +38,3 @@ def save(self): def delete(self): db.session.delete(self) db.session.commit() - - @staticmethod - def get_by_id(user_id): - return User.query.get(user_id) - - @staticmethod - def get_by_email(email): - return User.query.filter_by(email=email).first() - - @staticmethod - def get_all(): - return User.query.all() diff --git a/app/blueprints/auth/repositories.py b/app/blueprints/auth/repositories.py new file mode 100644 index 000000000..fa247f900 --- /dev/null +++ b/app/blueprints/auth/repositories.py @@ -0,0 +1,21 @@ +from app.blueprints.auth.models import User +from core.repositories.BaseRepository import BaseRepository + + +class UserRepository(BaseRepository): + def __init__(self): + super().__init__(User) + + def create(self, commit: bool = True, **kwargs): + password = kwargs.pop("password") + instance = self.model(**kwargs) + instance.set_password(password) + self.session.add(instance) + if commit: + self.session.commit() + else: + self.session.flush() + return instance + + def get_by_email(self, email: str): + return self.model.query.filter_by(email=email).first() diff --git a/app/blueprints/auth/routes.py b/app/blueprints/auth/routes.py index b8bfae92f..e42076ebe 100644 --- a/app/blueprints/auth/routes.py +++ b/app/blueprints/auth/routes.py @@ -1,60 +1,50 @@ -from flask import (render_template, redirect, url_for, request) +from flask import render_template, redirect, url_for, request from flask_login import current_user, login_user, logout_user from app.blueprints.auth import auth_bp from app.blueprints.auth.forms import SignupForm, LoginForm from app.blueprints.auth.services import AuthenticationService +from app.blueprints.profile.services import UserProfileService -from app.blueprints.profile.models import UserProfile + +authentication_service = AuthenticationService() +user_profile_service = UserProfileService() @auth_bp.route("/signup/", methods=["GET", "POST"]) def show_signup_form(): if current_user.is_authenticated: return redirect(url_for('public.index')) + form = SignupForm() - error = None if form.validate_on_submit(): - name = form.name.data - surname = form.surname.data email = form.email.data - password = form.password.data - - from app.blueprints.auth.models import User - user = User.get_by_email(email) - if user is not None: - error = f'Email {email} in use' - else: - # Create user - user = User(email=email) - user.set_password(password) - user.save() - - # Create user profile - profile = UserProfile(name=name, surname=surname) - profile.user_id = user.id - profile.save() - - # Log user - login_user(user, remember=True) - return redirect(url_for('public.index')) - return render_template("auth/signup_form.html", form=form, error=error) + if not authentication_service.is_email_available(email): + return render_template("auth/signup_form.html", form=form, error=f'Email {email} in use') + + try: + user = authentication_service.create_with_profile(**form.data) + except Exception as exc: + return render_template("auth/signup_form.html", form=form, error=f'Error creating user: {exc}') + + # Log user + login_user(user, remember=True) + return redirect(url_for('public.index')) + + return render_template("auth/signup_form.html", form=form) @auth_bp.route('/login', methods=['GET', 'POST']) def login(): if current_user.is_authenticated: return redirect(url_for('public.index')) + form = LoginForm() - if request.method == 'POST': - if form.validate_on_submit(): - email = form.email.data - password = form.password.data - if AuthenticationService.login(email, password): - return redirect(url_for('public.index')) - else: - error = 'Invalid credentials' - return render_template("auth/login_form.html", form=form, error=error) + if request.method == 'POST' and form.validate_on_submit(): + if authentication_service.login(form.email.data, form.password.data): + return redirect(url_for('public.index')) + + return render_template("auth/login_form.html", form=form, error='Invalid credentials') return render_template('auth/login_form.html', form=form) diff --git a/app/blueprints/auth/services.py b/app/blueprints/auth/services.py index 0032bbc6b..7ff368b25 100644 --- a/app/blueprints/auth/services.py +++ b/app/blueprints/auth/services.py @@ -1,14 +1,63 @@ from flask_login import login_user -from app.blueprints.auth.models import User +from app.blueprints.auth.repositories import UserRepository +from app.blueprints.profile.repositories import UserProfileRepository +from core.services.BaseService import BaseService -class AuthenticationService: +class AuthenticationService(BaseService): + def __init__(self): + super().__init__(UserRepository()) + self.user_profile_repository = UserProfileRepository() - @staticmethod - def login(email, password, remember=True): - user = User.get_by_email(email) + def login(self, email, password, remember=True): + user = self.repository.get_by_email(email) if user is not None and user.check_password(password): login_user(user, remember=remember) return True return False + + def is_email_available(self, email: str) -> bool: + return self.repository.get_by_email(email) is None + + def create_with_profile(self, **kwargs): + try: + email = kwargs.pop("email", None) + password = kwargs.pop("password", None) + name = kwargs.pop("name", None) + surname = kwargs.pop("surname", None) + + if not email: + raise ValueError("Email is required.") + if not password: + raise ValueError("Password is required.") + if not name: + raise ValueError("Name is required.") + if not surname: + raise ValueError("Surname is required.") + + user_data = { + "email": email, + "password": password + } + + profile_data = { + "name": name, + "surname": surname, + } + + user = self.create(commit=False, **user_data) + profile_data["user_id"] = user.id + self.user_profile_repository.create(**profile_data) + self.repository.session.commit() + except Exception as exc: + self.repository.session.rollback() + raise exc + return user + + def update_profile(self, user_profile_id, form): + if form.validate(): + updated_instance = self.update(user_profile_id, **form.data) + return updated_instance, None + + return None, form.errors diff --git a/app/blueprints/auth/tests/test_unit.py b/app/blueprints/auth/tests/test_unit.py index c4b08e36d..7ec22cf3f 100644 --- a/app/blueprints/auth/tests/test_unit.py +++ b/app/blueprints/auth/tests/test_unit.py @@ -1,12 +1,15 @@ import pytest from flask import url_for +from app.blueprints.auth.services import AuthenticationService +from app.blueprints.auth.repositories import UserRepository +from app.blueprints.profile.repositories import UserProfileRepository -@pytest.fixture(scope='module') + +@pytest.fixture(scope="module") def test_client(test_client): """ Extends the test_client fixture to add additional specific data for module testing. - for module testing (por example, new users) """ with test_client.application.app_context(): # Add HERE new elements to the database that you want to exist in the test context. @@ -17,33 +20,100 @@ def test_client(test_client): def test_login_success(test_client): - response = test_client.post('/login', data=dict( - email='test@example.com', - password='test1234' - ), follow_redirects=True) + response = test_client.post( + "/login", data=dict(email="test@example.com", password="test1234"), follow_redirects=True + ) - assert response.request.path != url_for('auth.login'), "Login was unsuccessful" + assert response.request.path != url_for("auth.login"), "Login was unsuccessful" - test_client.get('/logout', follow_redirects=True) + test_client.get("/logout", follow_redirects=True) def test_login_unsuccessful_bad_email(test_client): - response = test_client.post('/login', data=dict( - email='bademail@example.com', - password='test1234' - ), follow_redirects=True) + response = test_client.post( + "/login", data=dict(email="bademail@example.com", password="test1234"), follow_redirects=True + ) - assert response.request.path == url_for('auth.login'), "Login was unsuccessful" + assert response.request.path == url_for("auth.login"), "Login was unsuccessful" - test_client.get('/logout', follow_redirects=True) + test_client.get("/logout", follow_redirects=True) def test_login_unsuccessful_bad_password(test_client): - response = test_client.post('/login', data=dict( - email='test@example.com', - password='basspassword' - ), follow_redirects=True) + response = test_client.post( + "/login", data=dict(email="test@example.com", password="basspassword"), follow_redirects=True + ) + + assert response.request.path == url_for("auth.login"), "Login was unsuccessful" + + test_client.get("/logout", follow_redirects=True) + + +def test_signup_user_no_name(test_client): + response = test_client.post( + "/signup", data=dict(surname="Foo", email="test@example.com", password="test1234"), follow_redirects=True + ) + assert response.request.path == url_for("auth.show_signup_form"), "Signup was unsuccessful" + assert b"This field is required" in response.data, response.data + + +def test_signup_user_unsuccessful(test_client): + email = "test@example.com" + response = test_client.post( + "/signup", data=dict(name="Test", surname="Foo", email=email, password="test1234"), follow_redirects=True + ) + assert response.request.path == url_for("auth.show_signup_form"), "Signup was unsuccessful" + assert f"Email {email} in use".encode("utf-8") in response.data + + +def test_signup_user_successful(test_client): + response = test_client.post( + "/signup", + data=dict(name="Foo", surname="Example", email="foo@example.com", password="foo1234"), + follow_redirects=True, + ) + assert response.request.path == url_for("public.index"), "Signup was unsuccessful" + + +def test_service_create_with_profie_success(clean_database): + data = { + "name": "Test", + "surname": "Foo", + "email": "service_test@example.com", + "password": "test1234" + } + + AuthenticationService().create_with_profile(**data) + + assert UserRepository().count() == 1 + assert UserProfileRepository().count() == 1 + + +def test_service_create_with_profile_fail_no_email(clean_database): + data = { + "name": "Test", + "surname": "Foo", + "email": "", + "password": "1234" + } + + with pytest.raises(ValueError, match="Email is required."): + AuthenticationService().create_with_profile(**data) + + assert UserRepository().count() == 0 + assert UserProfileRepository().count() == 0 + + +def test_service_create_with_profile_fail_no_password(clean_database): + data = { + "name": "Test", + "surname": "Foo", + "email": "test@example.com", + "password": "" + } - assert response.request.path == url_for('auth.login'), "Login was unsuccessful" + with pytest.raises(ValueError, match="Password is required."): + AuthenticationService().create_with_profile(**data) - test_client.get('/logout', follow_redirects=True) + assert UserRepository().count() == 0 + assert UserProfileRepository().count() == 0 diff --git a/app/blueprints/conftest.py b/app/blueprints/conftest.py index bd62fa990..a5e004e9c 100644 --- a/app/blueprints/conftest.py +++ b/app/blueprints/conftest.py @@ -1,4 +1,5 @@ import pytest + from app import create_app, db from app.blueprints.auth.models import User @@ -8,6 +9,7 @@ def test_client(): flask_app = create_app('testing') with flask_app.test_client() as testing_client: with flask_app.app_context(): + db.drop_all() db.create_all() """ The test suite always includes the following user in order to avoid repetition @@ -21,6 +23,17 @@ def test_client(): db.drop_all() +@pytest.fixture(scope='function') +def clean_database(): + db.session.remove() + db.drop_all() + db.create_all() + yield + db.session.remove() + db.drop_all() + db.create_all() + + def login(test_client, email, password): """ Authenticates the user with the credentials provided. diff --git a/app/blueprints/dataset/models.py b/app/blueprints/dataset/models.py index 46c605d5c..6deff0f05 100644 --- a/app/blueprints/dataset/models.py +++ b/app/blueprints/dataset/models.py @@ -1,12 +1,11 @@ +import os from datetime import datetime +from enum import Enum from flask import request - -from app import db -from enum import Enum from sqlalchemy import Enum as SQLAlchemyEnum -import os +from app import db class PublicationType(Enum): diff --git a/app/blueprints/dataset/repositories.py b/app/blueprints/dataset/repositories.py new file mode 100644 index 000000000..5930402db --- /dev/null +++ b/app/blueprints/dataset/repositories.py @@ -0,0 +1,155 @@ +import re +import unidecode +from typing import Optional + +from sqlalchemy import or_, any_ + +from app.blueprints.dataset.models import ( + Author, + DSDownloadRecord, + DSMetaData, + DSViewRecord, + DataSet, + FMMetaData, + FeatureModel, + File, + FileDownloadRecord, + FileViewRecord, + PublicationType, +) +from core.repositories.BaseRepository import BaseRepository + + +class AuthorRepository(BaseRepository): + def __init__(self): + super().__init__(Author) + + +class DSDownloadRecordRepository(BaseRepository): + def __init__(self): + super().__init__(DSDownloadRecord) + + def total_dataset_downloads(self) -> int: + return self.count() + + +class DSMetaDataRepository(BaseRepository): + def __init__(self): + super().__init__(DSMetaData) + + def filter_by_doi(self, doi: str) -> Optional[DSMetaData]: + return self.model.query.filter_by(dataset_doi=doi).first() + + +class DSViewRecordRepository(BaseRepository): + def __init__(self): + super().__init__(DSViewRecord) + + def total_dataset_views(self) -> int: + return self.count() + + +class DataSetRepository(BaseRepository): + def __init__(self): + super().__init__(DataSet) + + def get_synchronized(self, current_user_id: int) -> DataSet: + return ( + self.model.query.join(DSMetaData) + .filter(DataSet.user_id == current_user_id, DSMetaData.dataset_doi.isnot(None)) + .order_by(self.model.created_at.desc()) + .all() + ) + + def get_unsynchronized(self, current_user_id: int) -> DataSet: + return ( + self.model.query.join(DSMetaData) + .filter(DataSet.user_id == current_user_id, DSMetaData.dataset_doi.is_(None)) + .order_by(self.model.created_at.desc()) + .all() + ) + + def filter(self, query="", sorting="newest", publication_type="any", tags=[], **kwargs): + # Normalize and remove unwanted characters + normalized_query = unidecode.unidecode(query).lower() + cleaned_query = re.sub(r'[,.":\'()\[\]^;!¡¿?]', "", normalized_query) + + filters = [] + for word in cleaned_query.split(): + filters.append(DSMetaData.title.ilike(f"%{word}%")) + filters.append(DSMetaData.description.ilike(f"%{word}%")) + filters.append(Author.name.ilike(f"%{word}%")) + filters.append(Author.affiliation.ilike(f"%{word}%")) + filters.append(Author.orcid.ilike(f"%{word}%")) + filters.append(FMMetaData.uvl_filename.ilike(f"%{word}%")) + filters.append(FMMetaData.title.ilike(f"%{word}%")) + filters.append(FMMetaData.description.ilike(f"%{word}%")) + filters.append(FMMetaData.publication_type.ilike(f"%{word}%")) + filters.append(FMMetaData.publication_doi.ilike(f"%{word}%")) + filters.append(FMMetaData.tags.ilike(f"%{word}%")) + filters.append(DSMetaData.tags.ilike(f"%{word}%")) + + datasets = ( + self.model.query.join(DSMetaData).join(Author).join(FeatureModel).join(FMMetaData).filter(or_(*filters)) + ) + + if publication_type != "any": + matching_type = None + for member in PublicationType: + if member.value.lower() == publication_type: + matching_type = member + break + + if matching_type is not None: + datasets = datasets.filter(DSMetaData.publication_type == matching_type.name) + + if tags: + datasets = datasets.filter(DSMetaData.tags.ilike(any_(f"%{tag}%" for tag in tags))) + + # Order by created_at + if sorting == "oldest": + datasets = datasets.order_by(self.model.created_at.asc()) + else: + datasets = datasets.order_by(self.model.created_at.desc()) + + return datasets.all() + + def latest_synchronized(self): + return ( + self.model.query.join(DSMetaData) + .filter(DSMetaData.dataset_doi.isnot(None)) + .order_by(self.model.created_at.desc()) + .limit(5) + .all() + ) + + +class FMMetaDataRepository(BaseRepository): + def __init__(self): + super().__init__(FMMetaData) + + +class FeatureModelRepository(BaseRepository): + def __init__(self): + super().__init__(FeatureModel) + + +class FileRepository(BaseRepository): + def __init__(self): + super().__init__(File) + + +class FileDownloadRecordRepository(BaseRepository): + def __init__(self): + super().__init__(FileDownloadRecord) + + def total_feature_model_downloads(self) -> int: + return self.count() + + +class FileViewRecordRepository(BaseRepository): + def __init__(self): + super().__init__(FileViewRecord) + + def total_feature_model_views(self) -> int: + return self.count() diff --git a/app/blueprints/dataset/routes.py b/app/blueprints/dataset/routes.py index 0625adef5..38cb14e47 100644 --- a/app/blueprints/dataset/routes.py +++ b/app/blueprints/dataset/routes.py @@ -4,76 +4,105 @@ import shutil import tempfile import uuid - -from datetime import datetime +from datetime import datetime, timezone from zipfile import ZipFile + from app import db -from flask import render_template, request, jsonify, send_from_directory, abort, \ - current_app, make_response +from flask import ( + render_template, + request, + jsonify, + send_from_directory, + current_app, + make_response, + abort, +) from flask_login import login_required, current_user import app from app.blueprints.dataset.forms import DataSetForm -from app.blueprints.dataset.models import DataSet, FileViewRecord, FeatureModel, File, FMMetaData, DSMetaData, Author, \ - PublicationType, DSDownloadRecord, DSViewRecord, FileDownloadRecord +from app.blueprints.dataset.models import ( + DSDownloadRecord, + DSViewRecord, + DataSet, + File, + FileDownloadRecord, + FileViewRecord, + PublicationType +) from app.blueprints.dataset import dataset_bp -from app.blueprints.zenodo.services import test_full_zenodo_connection, zenodo_create_new_deposition, \ - zenodo_upload_file, zenodo_publish_deposition, zenodo_get_doi - - -@dataset_bp.route('/zenodo/test', methods=['GET']) -def zenodo_test() -> dict: - return test_full_zenodo_connection() - - -@dataset_bp.route('/dataset/upload', methods=['GET', 'POST']) +from app.blueprints.dataset.services import ( + AuthorService, + DSDownloadRecordService, + DSMetaDataService, + DSViewRecordService, + DataSetService, + FMMetaDataService, + FeatureModelService, + FileService, + FileDownloadRecordService, +) +from app.blueprints.zenodo.services import ZenodoService + + +dataset_service = DataSetService() +author_service = AuthorService() +dsmetadata_service = DSMetaDataService() +zenodo_service = ZenodoService() + + +@dataset_bp.route("/dataset/upload", methods=["GET", "POST"]) @login_required def create_dataset(): form = DataSetForm() - if request.method == 'POST': - + if request.method == "POST": try: - # get JSON from frontend - form_data_json = request.form.get('formData') + form_data_json = request.form.get("formData") form_data_dict = json.loads(form_data_json) # get dicts - basic_info_data = form_data_dict["basic_info_form"] uploaded_models_data = form_data_dict["uploaded_models_form"] # create dataset - dataset = create_dataset_in_db(basic_info_data) + dataset = create_dataset_in_db(form_data_dict["basic_info_form"]) # send dataset as deposition to Zenodo - zenodo_response_json = zenodo_create_new_deposition(dataset) - - response_data = json.dumps(zenodo_response_json) - data = json.loads(response_data) - if data.get('conceptrecid'): + try: + zenodo_response_json = zenodo_service.create_new_deposition(dataset) + response_data = json.dumps(zenodo_response_json) + data = json.loads(response_data) + except Exception: + data = {} + zenodo_response_json = {} - deposition_id = data.get('id') + if data.get("conceptrecid"): + deposition_id = data.get("id") # update dataset with deposition id in Zenodo - dataset.ds_meta_data.deposition_id = deposition_id - app.db.session.commit() + dsmetadata_service.update( + dataset.ds_meta_data_id, deposition_id=deposition_id + ) # create feature models - feature_models = create_feature_models_in_db(dataset, uploaded_models_data) + feature_models = create_feature_models_in_db( + dataset, uploaded_models_data + ) try: # iterate for each feature model (one feature model = one request to Zenodo for feature_model in feature_models: - zenodo_upload_file(deposition_id, feature_model) + zenodo_service.upload_file(deposition_id, feature_model) # publish deposition - zenodo_publish_deposition(deposition_id) + zenodo_service.publish_deposition(deposition_id) # update DOI - deposition_doi = zenodo_get_doi(deposition_id) - dataset.ds_meta_data.dataset_doi = deposition_doi - app.db.session.commit() + deposition_doi = zenodo_service.get_doi(deposition_id) + dsmetadata_service.update( + dataset.ds_meta_data_id, dataset_doi=deposition_doi + ) except Exception: pass @@ -84,182 +113,147 @@ def create_dataset(): # it has not been possible to create the deposition in Zenodo, so we save everything locally # create feature models - feature_models = create_feature_models_in_db(dataset, uploaded_models_data) + feature_models = create_feature_models_in_db( + dataset, uploaded_models_data + ) # move feature models permanently move_feature_models(dataset.id, feature_models) - pass - return jsonify({'message': zenodo_response_json}), 200 + return jsonify({"message": zenodo_response_json}), 200 except Exception as e: - return jsonify({'message': str(e)}), 500 + return jsonify({"message": str(e)}), 500 # Delete temp folder - file_path = os.path.join(app.upload_folder_name(), 'temp', str(current_user.id)) + file_path = os.path.join(app.upload_folder_name(), "temp", str(current_user.id)) if os.path.exists(file_path) and os.path.isdir(file_path): shutil.rmtree(file_path) - return render_template('dataset/upload_dataset.html', form=form) + return render_template("dataset/upload_dataset.html", form=form) -@dataset_bp.route('/dataset/list', methods=['GET', 'POST']) +@dataset_bp.route("/dataset/list", methods=["GET", "POST"]) @login_required def list_dataset(): - # synchronized datasets - datasets = DataSet.query.join(DSMetaData).filter( - DataSet.user_id == current_user.id, - DSMetaData.dataset_doi.isnot(None) - ).order_by(DataSet.created_at.desc()).all() - - # local datasets - local_datasets = DataSet.query.join(DSMetaData).filter( - DataSet.user_id == current_user.id, - DSMetaData.dataset_doi.is_(None) - ).order_by(DataSet.created_at.desc()).all() - - return render_template('dataset/list_datasets.html', datasets=datasets, local_datasets=local_datasets) + return render_template( + "dataset/list_datasets.html", + datasets=dataset_service.get_synchronized(current_user.id), + local_datasets=dataset_service.get_unsynchronized(current_user.id), + ) def create_dataset_in_db(basic_info_data): - # get dataset metadata - title = basic_info_data["title"][0] - description = basic_info_data["description"][0] - publication_type = basic_info_data["publication_type"][0] - publication_doi = basic_info_data["publication_doi"][0] - tags = basic_info_data["tags"][0] - - # create dataset metadata - ds_meta_data = DSMetaData( - title=title, - description=description, - publication_type=PublicationType(publication_type), - publication_doi=publication_doi, - tags=tags - ) - app.db.session.add(ds_meta_data) - app.db.session.commit() + ds_data = { + "title": basic_info_data["title"][0], + "description": basic_info_data["description"][0], + "publication_type": PublicationType(basic_info_data["publication_type"][0]), + "publication_doi": basic_info_data["publication_doi"][0], + "tags": basic_info_data["tags"][0], + } + ds_meta_data = dsmetadata_service.create(**ds_data) # create dataset metadata authors - # I always add myself - author = Author( - name=f"{current_user.profile.surname}, {current_user.profile.name}", - affiliation=current_user.profile.affiliation if hasattr(current_user.profile, 'affiliation') else None, - orcid=current_user.profile.orcid if hasattr(current_user.profile, 'orcid') else None, - ds_meta_data_id=ds_meta_data.id - ) - app.db.session.add(author) - app.db.session.commit() + author_data = { + "name": f"{current_user.profile.surname}, {current_user.profile.name}", + "affiliation": current_user.profile.affiliation + if hasattr(current_user.profile, "affiliation") + else None, + "orcid": current_user.profile.orcid + if hasattr(current_user.profile, "orcid") + else None, + "ds_meta_data_id": ds_meta_data.id, + } + author_service.create(**author_data) # how many authors are there? if "author_name" in basic_info_data: number_of_authors = len(basic_info_data["author_name"]) for i in range(number_of_authors): - author_name = basic_info_data["author_name"][i] - author_affiliation = basic_info_data["author_affiliation"][i] - author_orcid = basic_info_data["author_orcid"][i] - - author = Author( - name=author_name, - affiliation=author_affiliation, - orcid=author_orcid, - ds_meta_data_id=ds_meta_data.id - ) - app.db.session.add(author) - app.db.session.commit() + extra_author = { + "name": basic_info_data["author_name"][i], + "affiliation": basic_info_data["author_affiliation"][i], + "orcid": basic_info_data["author_orcid"][i], + "ds_meta_data_id": ds_meta_data.id, + } + author_service.create(**extra_author) # create dataset - dataset = DataSet(user_id=current_user.id, ds_meta_data_id=ds_meta_data.id) - app.db.session.add(dataset) - app.db.session.commit() - - return dataset + return dataset_service.create( + user_id=current_user.id, ds_meta_data_id=ds_meta_data.id + ) def create_feature_models_in_db(dataset: DataSet, uploaded_models_data: dict): feature_models = [] - if "uvl_identifier" in uploaded_models_data: - - number_of_models = len(uploaded_models_data["uvl_identifier"]) - - for i in range(number_of_models): - - # get feature model metadata - uvl_identifier = uploaded_models_data["uvl_identifier"][i] - uvl_filename = uploaded_models_data["uvl_filename"][i] - title = uploaded_models_data["title"][i] - description = uploaded_models_data["description"][i] - uvl_publication_type = uploaded_models_data["uvl_publication_type"][i] - publication_doi = uploaded_models_data["publication_doi"][i] - tags = uploaded_models_data["tags"][i] - uvl_version = uploaded_models_data["uvl_version"][i] - - # create feature model metadata - feature_model_metadata = FMMetaData( - uvl_filename=uvl_filename, - title=title, - description=description, - publication_type=PublicationType(uvl_publication_type), - publication_doi=publication_doi, - tags=tags, - uvl_version=uvl_version - ) - app.db.session.add(feature_model_metadata) - app.db.session.commit() + for i, uvl_identifier in enumerate(uploaded_models_data.get("uvl_identifier", [])): + uvl_filename = uploaded_models_data["uvl_filename"][i] - # create feature model - feature_model = FeatureModel( - data_set_id=dataset.id, - fm_meta_data_id=feature_model_metadata.id - ) - app.db.session.add(feature_model) - app.db.session.commit() - - # associated authors in feature model - if f"author_name_{uvl_identifier}" in uploaded_models_data.keys(): - number_of_authors_in_model = len(uploaded_models_data[f"author_name_{uvl_identifier}"]) - for a in range(number_of_authors_in_model): - author = Author( - name=uploaded_models_data[f"author_name_{uvl_identifier}"][a], - affiliation=uploaded_models_data[f"author_affiliation_{uvl_identifier}"][a], - orcid=uploaded_models_data[f"author_orcid_{uvl_identifier}"][a], - fm_meta_data_id=feature_model_metadata.id - ) - app.db.session.add(author) - app.db.session.commit() - - # associated files in feature model - user_id = current_user.id - file_path = os.path.join(app.upload_folder_name(), 'temp', str(user_id), uvl_filename) - checksum, size = calculate_checksum_and_size(file_path) - file = File( - name=uvl_filename, - checksum=checksum, - size=size, - feature_model_id=feature_model.id + # create feature model metadata + feature_model_metadata = FMMetaDataService().create( + uvl_filename=uvl_filename, + title=uploaded_models_data["title"][i], + description=uploaded_models_data["description"][i], + publication_type=PublicationType( + uploaded_models_data["uvl_publication_type"][i] + ), + publication_doi=uploaded_models_data["publication_doi"][i], + tags=uploaded_models_data["tags"][i], + uvl_version=uploaded_models_data["uvl_version"][i], + ) + + # create feature model + feature_model = FeatureModelService().create( + data_set_id=dataset.id, + fm_meta_data_id=feature_model_metadata.id, + ) + + # associated authors in feature model + for idx, author_name in enumerate( + uploaded_models_data.get(f"author_name_{uvl_identifier}", []) + ): + author_service.create( + name=author_name, + affiliation=uploaded_models_data[ + f"author_affiliation_{uvl_identifier}" + ][idx], + orcid=uploaded_models_data[f"author_orcid_{uvl_identifier}"][idx], + fm_meta_data_id=feature_model_metadata.id, ) - app.db.session.add(file) - app.db.session.commit() - feature_models.append(feature_model) + # associated files in feature model + user_id = current_user.id + file_path = os.path.join( + app.upload_folder_name(), "temp", str(user_id), uvl_filename + ) + checksum, size = calculate_checksum_and_size(file_path) + + FileService().create( + name=uvl_filename, + checksum=checksum, + size=size, + feature_model_id=feature_model.id, + ) + + feature_models.append(feature_model) return feature_models def calculate_checksum_and_size(file_path): file_size = os.path.getsize(file_path) - with open(file_path, 'rb') as file: + with open(file_path, "rb") as file: content = file.read() hash_md5 = hashlib.md5(content).hexdigest() return hash_md5, file_size -def move_feature_models(dataset_id, feature_models, user=None): - user_id = current_user.id if user is None else user.id - source_dir = f'uploads/temp/{user_id}/' - dest_dir = f'uploads/user_{user_id}/dataset_{dataset_id}/' +def move_feature_models(dataset_id, feature_models): + user_id = current_user.id + source_dir = f"uploads/temp/{user_id}/" + dest_dir = f"uploads/user_{user_id}/dataset_{dataset_id}/" os.makedirs(dest_dir, exist_ok=True) @@ -268,98 +262,112 @@ def move_feature_models(dataset_id, feature_models, user=None): shutil.move(os.path.join(source_dir, uvl_filename), dest_dir) -@dataset_bp.route('/dataset/file/upload', methods=['POST']) +@dataset_bp.route("/dataset/file/upload", methods=["POST"]) @login_required def upload(): - file = request.files['file'] + file = request.files["file"] user_id = current_user.id - temp_folder = os.path.join(app.upload_folder_name(), 'temp', str(user_id)) - - if file and file.filename.endswith('.uvl'): - - # create temp folder - if not os.path.exists(temp_folder): - os.makedirs(temp_folder) - - file_path = os.path.join(temp_folder, file.filename) + temp_folder = os.path.join(app.upload_folder_name(), "temp", str(user_id)) + + if not file or not file.filename.endswith(".uvl"): + return jsonify({"message": "No valid file"}), 400 + + # create temp folder + if not os.path.exists(temp_folder): + os.makedirs(temp_folder) + + file_path = os.path.join(temp_folder, file.filename) + + if os.path.exists(file_path): + # Generate unique filename (by recursion) + base_name, extension = os.path.splitext(file.filename) + i = 1 + while os.path.exists( + os.path.join(temp_folder, f"{base_name} ({i}){extension}") + ): + i += 1 + new_filename = f"{base_name} ({i}){extension}" + file_path = os.path.join(temp_folder, new_filename) + else: + new_filename = file.filename - if os.path.exists(file_path): - # Generate unique filename (by recursion) - base_name, extension = os.path.splitext(file.filename) - i = 1 - while os.path.exists(os.path.join(temp_folder, f"{base_name} ({i}){extension}")): - i += 1 - new_filename = f"{base_name} ({i}){extension}" - file_path = os.path.join(temp_folder, new_filename) + try: + file.save(file_path) + if True: + return ( + jsonify( + { + "message": "UVL uploaded and validated successfully", + "filename": new_filename, + } + ), + 200, + ) else: - new_filename = file.filename - - try: - file.save(file_path) - if True: - return jsonify({ - 'message': 'UVL uploaded and validated successfully', - 'filename': new_filename - }), 200 - else: - return jsonify({'message': 'No valid model'}), 400 - except Exception as e: - return jsonify({'message': str(e)}), 500 - - else: - return jsonify({'message': 'No valid file'}), 400 + return jsonify({"message": "No valid model"}), 400 + except Exception as e: + return jsonify({"message": str(e)}), 500 -@dataset_bp.route('/dataset/file/delete', methods=['POST']) +@dataset_bp.route("/dataset/file/delete", methods=["POST"]) def delete(): data = request.get_json() - filename = data.get('file') + filename = data.get("file") user_id = current_user.id - temp_folder = os.path.join(app.upload_folder_name(), 'temp', str(user_id)) + temp_folder = os.path.join(app.upload_folder_name(), "temp", str(user_id)) filepath = os.path.join(temp_folder, filename) if os.path.exists(filepath): os.remove(filepath) - return jsonify({'message': 'File deleted successfully'}) - else: - return jsonify({'error': 'Error: File not found'}) + return jsonify({"message": "File deleted successfully"}) + return jsonify({"error": "Error: File not found"}) -@dataset_bp.route('/dataset/download/', methods=['GET']) + +@dataset_bp.route("/dataset/download/", methods=["GET"]) def download_dataset(dataset_id): - dataset = DataSet.query.get_or_404(dataset_id) + dataset = dataset_service.get_or_404(dataset_id) file_path = f"uploads/user_{dataset.user_id}/dataset_{dataset.id}/" temp_dir = tempfile.mkdtemp() - zip_path = os.path.join(temp_dir, f'dataset_{dataset_id}.zip') + zip_path = os.path.join(temp_dir, f"dataset_{dataset_id}.zip") - with ZipFile(zip_path, 'w') as zipf: + with ZipFile(zip_path, "w") as zipf: for subdir, dirs, files in os.walk(file_path): for file in files: full_path = os.path.join(subdir, file) relative_path = os.path.relpath(full_path, file_path) - zipf.write(full_path, arcname=os.path.join(os.path.basename(zip_path[:-4]), relative_path)) + zipf.write( + full_path, + arcname=os.path.join( + os.path.basename(zip_path[:-4]), relative_path + ), + ) - user_cookie = request.cookies.get('download_cookie') + user_cookie = request.cookies.get("download_cookie") if not user_cookie: - user_cookie = str(uuid.uuid4()) # Generate a new unique identifier if it does not exist + user_cookie = str( + uuid.uuid4() + ) # Generate a new unique identifier if it does not exist # Save the cookie to the user's browser - resp = make_response(send_from_directory( - temp_dir, - f'dataset_{dataset_id}.zip', - as_attachment=True, - mimetype='application/zip' - )) - resp.set_cookie('download_cookie', user_cookie) + resp = make_response( + send_from_directory( + temp_dir, + f"dataset_{dataset_id}.zip", + as_attachment=True, + mimetype="application/zip", + ) + ) + resp.set_cookie("download_cookie", user_cookie) else: resp = send_from_directory( temp_dir, - f'dataset_{dataset_id}.zip', + f"dataset_{dataset_id}.zip", as_attachment=True, - mimetype='application/zip' + mimetype="application/zip", ) # Check if the download record already exists for this cookie @@ -371,25 +379,22 @@ def download_dataset(dataset_id): if not existing_record: # Record the download in your database - download_record = DSDownloadRecord( + DSDownloadRecordService().create( user_id=current_user.id if current_user.is_authenticated else None, dataset_id=dataset_id, - download_date=datetime.now(), - download_cookie=user_cookie + download_date=datetime.now(timezone.utc), + download_cookie=user_cookie, ) - app.db.session.add(download_record) - app.db.session.commit() - return resp -@dataset_bp.route('/dataset/view/', methods=['GET']) +@dataset_bp.route("/dataset/view/", methods=["GET"]) def view_dataset(dataset_id): - dataset = DataSet.query.get_or_404(dataset_id) + dataset = dataset_service.get_or_404(dataset_id) # Get the cookie from the request or generate a new one if it does not exist - user_cookie = request.cookies.get('view_cookie') + user_cookie = request.cookies.get("view_cookie") if not user_cookie: user_cookie = str(uuid.uuid4()) @@ -402,25 +407,23 @@ def view_dataset(dataset_id): if not existing_record: # Record the view in your database - view_record = DSViewRecord( + DSViewRecordService().create( user_id=current_user.id if current_user.is_authenticated else None, dataset_id=dataset_id, - view_date=datetime.now(), - view_cookie=user_cookie + view_date=datetime.now(timezone.utc), + view_cookie=user_cookie, ) - app.db.session.add(view_record) - app.db.session.commit() # Save the cookie to the user's browser - resp = make_response(render_template('dataset/view_dataset.html', dataset=dataset)) - resp.set_cookie('view_cookie', user_cookie) + resp = make_response(render_template("dataset/view_dataset.html", dataset=dataset)) + resp.set_cookie("view_cookie", user_cookie) return resp -@dataset_bp.route('/file/download/', methods=['GET']) +@dataset_bp.route("/file/download/", methods=["GET"]) def download_file(file_id): - file = File.query.get_or_404(file_id) + file = FileService().get_or_404(file_id) filename = file.name directory_path = f"uploads/user_{file.feature_model.data_set.user_id}/dataset_{file.feature_model.data_set_id}/" @@ -428,7 +431,7 @@ def download_file(file_id): file_path = os.path.join(parent_directory_path, directory_path) # Get the cookie from the request or generate a new one if it does not exist - user_cookie = request.cookies.get('file_download_cookie') + user_cookie = request.cookies.get("file_download_cookie") if not user_cookie: user_cookie = str(uuid.uuid4()) @@ -441,18 +444,18 @@ def download_file(file_id): if not existing_record: # Record the download in your database - download_record = FileDownloadRecord( + FileDownloadRecordService().create( user_id=current_user.id if current_user.is_authenticated else None, file_id=file_id, - download_date=datetime.now(), - download_cookie=user_cookie + download_date=datetime.now(timezone.utc), + download_cookie=user_cookie, ) - app.db.session.add(download_record) - app.db.session.commit() # Save the cookie to the user's browser - resp = make_response(send_from_directory(directory=file_path, path=filename, as_attachment=True)) - resp.set_cookie('file_download_cookie', user_cookie) + resp = make_response( + send_from_directory(directory=file_path, path=filename, as_attachment=True) + ) + resp.set_cookie("file_download_cookie", user_cookie) return resp @@ -506,286 +509,36 @@ def view_file(file_id): return jsonify({'success': False, 'error': str(e)}), 500 -''' - API ENDPOINTS FOR DATASET MODEL -''' - -''' -@dataset_bp.route('/api/v1/dataset/', methods=['GET']) -def get_all_dataset(): - datasets = DataSet.query.order_by(DataSet.created_at.desc()).all() - dataset_list = [dataset.to_dict() for dataset in datasets] - return jsonify(dataset_list) - - -@dataset_bp.route('/api/v1/dataset/', methods=['GET']) -def get_dataset(dataset_id): - dataset = DataSet.query.get_or_404(dataset_id) - return dataset.to_dict() - - -@dataset_bp.route('/api/v1/dataset/', methods=['POST']) -def api_create_dataset(): - """ - ENDPOINT FOR CREATE DATASET - """ - - """ - PART 1: GET BASIC DATA - """ - - user = app.get_user_by_token("BLABLABLA") # TODO - data = json.loads(request.files['json'].read()) - temp_folder = os.path.join(app.upload_folder_name(), 'temp', str(user.id)) - - # Delete the existing temp_folder if it exists - if os.path.exists(temp_folder): - shutil.rmtree(temp_folder) - - # Create the new temp_folder - os.makedirs(temp_folder) - - info = data['info'] - models = data['models'] - - """ - PART 2: SAVE BASIC DATA - """ - ds_meta_data = _create_ds_meta_data(info=info) - _create_authors(info=info, ds_meta_data=ds_meta_data) - dataset = _create_dataset(user=user, ds_meta_data=ds_meta_data) - - """ - PART 3: SAVE FILES IN TEMP FOLDER - """ - files = request.files.to_dict() - for filename, file in files.items(): - if filename != 'json': - - if file and filename.endswith('.uvl'): - - # create temporal folder for this user - if not os.path.exists(temp_folder): - os.makedirs(temp_folder) - - try: - filename = os.path.basename(file.filename) - file.save(os.path.join(temp_folder, filename)) - # TODO: Change valid model function - if True: - continue # TODO - else: - dataset.delete() - return jsonify({'message': f'{filename} is not a valid model'}), 400 - except Exception as e: - dataset.delete() - return jsonify({'Exception in save files in temp folder': str(e)}), 500 - - else: - dataset.delete() - return jsonify({'message': f'{filename} is not a valid extension'}), 400 - - """ - PART 4: SEND BASIC DATA TO ZENODO - """ - zenodo_response_json = zenodo_create_new_deposition(dataset) - response_data = json.dumps(zenodo_response_json) - zenodo_json_data = json.loads(response_data) - - """ - PART 5: CREATE FEATURE MODELS - """ - feature_models = _create_feature_models(dataset=dataset, models=models, user=user) - - if zenodo_json_data.get('conceptrecid'): - - # update dataset with deposition id in Zenodo - deposition_id = zenodo_json_data.get('id') - dataset.ds_meta_data.deposition_id = deposition_id - app.db.session.commit() - - """ - PART 6: SEND FILES TO ZENODO AND PUBLISH - """ - try: - # iterate for each feature model (one feature model = one request to Zenodo - try: - for feature_model in feature_models: - zenodo_upload_file(deposition_id, feature_model, user=user) - - # Wait for 0.6 seconds before the next API call to ensure we do not exceed - # the rate limit of 100 requests per minute. This is because 60 seconds (1 minute) - # divided by 100 requests equals 0.6 seconds per request. - time.sleep(0.6) - - except Exception as e: - logging.error("Exception occurred during file upload", exc_info=True) - return jsonify({'exception': str(e)}), 500 - - # publish deposition - try: - zenodo_publish_deposition(deposition_id) - except Exception as e: - logging.error("Exception occurred during publish deposition", exc_info=True) - return jsonify({'exception': str(e)}), 500 - - # update DOI - try: - deposition_doi = zenodo_get_doi(deposition_id) - dataset.ds_meta_data.dataset_doi = deposition_doi - app.db.session.commit() - except Exception as e: - logging.error("Exception occurred during update DOI", exc_info=True) - return jsonify({'exception': str(e)}), 500 - - except Exception as e: - logging.error("Exception occurred", exc_info=True) - return jsonify({'exception': str(e)}), 500 - - """ - PART 7: MOVE FEATURE MODELS PERMANENTLY - """ - move_feature_models(dataset.id, feature_models, user=user) - - return jsonify(dataset.to_dict()), 200 - - -def _create_ds_meta_data(info: dict) -> DSMetaData: - ds_meta_data = DSMetaData( - title=info["title"], - description=info["description"], - publication_type=PublicationType(info["publication_type"]), - publication_doi=info["publication_doi"], - tags=','.join(tag.strip() for tag in info['tags']) - ) - app.db.session.add(ds_meta_data) - app.db.session.commit() - - return ds_meta_data - - -def _create_authors(info: dict, ds_meta_data: DSMetaData) -> List[Author]: - authors = [] - - authors_info = info.get("authors") - if authors_info: - for author_info in authors_info: - author = Author( - name=author_info.get("name"), - affiliation=author_info.get("affiliation"), - orcid=author_info.get("orcid", None), - ds_meta_data_id=ds_meta_data.id - ) - authors.append(author) - app.db.session.add(author) - - app.db.session.commit() - - return authors - - -def _create_dataset(user: User, ds_meta_data: DSMetaData) -> DataSet: - dataset = DataSet(user_id=user.id, ds_meta_data_id=ds_meta_data.id) - app.db.session.add(dataset) - app.db.session.commit() - - return dataset - - -def _create_feature_models(dataset: DataSet, models: dict, user: User) -> List[FeatureModel]: - feature_models = [] - - for model in models: - - filename = os.path.basename(model['filename']) # only name of file with .uvl extension - title = model.get('title', '') - description = model.get('description', '') - publication_type = model.get('publication_type', 'none') - publication_doi = model.get('publication_doi', '') - tags = ','.join(tag.strip() for tag in model.get('tags', [])) - - # create feature model metadata - feature_model_metadata = FMMetaData( - uvl_filename=filename, - title=title, - description=description, - publication_type=publication_type, - publication_doi=publication_doi, - tags=tags - ) - app.db.session.add(feature_model_metadata) - app.db.session.commit() - - # associated authors in feature model - if 'authors' in model and isinstance(model['authors'], list): - for author_data in model['authors']: - if 'name' in author_data: - author = Author( - name=author_data.get('name'), - affiliation=author_data.get('affiliation'), - orcid=author_data.get('orcid'), - fm_meta_data_id=feature_model_metadata.id - ) - app.db.session.add(author) - app.db.session.commit() - - # create feature model - feature_model = FeatureModel( - data_set_id=dataset.id, - fm_meta_data_id=feature_model_metadata.id - ) - app.db.session.add(feature_model) - app.db.session.commit() - - # associated files in feature model - user_id = user.id - file_path = os.path.join(app.upload_folder_name(), 'temp', str(user_id), filename) - checksum, size = calculate_checksum_and_size(file_path) - file = File( - name=filename, - checksum=checksum, - size=size, - feature_model_id=feature_model.id - ) - app.db.session.add(file) - app.db.session.commit() - - feature_models.append(feature_model) - - return feature_models - -''' - - -@dataset_bp.route('/doi//', methods=['GET']) +@dataset_bp.route("/doi//", methods=["GET"]) def subdomain_index(doi): # Busca el dataset por DOI - ds_meta_data = DSMetaData.query.filter_by(dataset_doi=doi).first() - if ds_meta_data: - dataset = ds_meta_data.data_set + ds_meta_data = dsmetadata_service.filter_by_doi(doi) + if not ds_meta_data: + abort(404) - if dataset: - dataset_id = dataset.id - user_cookie = request.cookies.get('view_cookie', str(uuid.uuid4())) + dataset = ds_meta_data.data_set + if dataset: + dataset_id = dataset.id + user_cookie = request.cookies.get("view_cookie", str(uuid.uuid4())) - # Registra la vista del dataset - view_record = DSViewRecord( - user_id=current_user.id if current_user.is_authenticated else None, - dataset_id=dataset_id, - view_date=datetime.now(), - view_cookie=user_cookie - ) - app.db.session.add(view_record) - app.db.session.commit() - - # Prepara la respuesta y establece la cookie - resp = make_response(render_template('dataset/view_dataset.html', dataset=dataset)) - resp.set_cookie('view_cookie', user_cookie, max_age=30 * 24 * 60 * 60) # Ejemplo: cookie expira en 30 días + # Registra la vista del dataset + DSViewRecordService().create( + user_id=current_user.id if current_user.is_authenticated else None, + dataset_id=dataset_id, + view_date=datetime.now(timezone.utc), + view_cookie=user_cookie, + ) - return resp - else: - # Aquí puedes manejar el caso de que el DOI no corresponda a un dataset existente - # Por ejemplo, mostrar un error 404 o redirigir a una página de error - return "Dataset no encontrado", 404 + # Prepara la respuesta y establece la cookie + resp = make_response( + render_template("dataset/view_dataset.html", dataset=dataset) + ) + resp.set_cookie( + "view_cookie", user_cookie, max_age=30 * 24 * 60 * 60 + ) # Ejemplo: cookie expira en 30 días - abort(404) + return resp + else: + # Aquí puedes manejar el caso de que el DOI no corresponda a un dataset existente + # Por ejemplo, mostrar un error 404 o redirigir a una página de error + return "Dataset not found", 404 diff --git a/app/blueprints/dataset/services.py b/app/blueprints/dataset/services.py new file mode 100644 index 000000000..ded4d974e --- /dev/null +++ b/app/blueprints/dataset/services.py @@ -0,0 +1,107 @@ +from typing import Optional + +from app.blueprints.dataset.models import DataSet, DSMetaData +from app.blueprints.dataset.repositories import ( + AuthorRepository, + DSDownloadRecordRepository, + DSMetaDataRepository, + DSViewRecordRepository, + DataSetRepository, + FMMetaDataRepository, + FeatureModelRepository, + FileRepository, + FileDownloadRecordRepository, + FileViewRecordRepository +) +from core.services.BaseService import BaseService + + +class DataSetService(BaseService): + def __init__(self): + super().__init__(DataSetRepository()) + self.feature_model_service = FeatureModelService() + self.author_repository = AuthorRepository() + self.dsmetadata_repository = DSMetaDataRepository() + self.dsdownloadrecord_repository = DSDownloadRecordRepository() + self.filedownloadrecord_repository = FileDownloadRecordRepository() + self.dsviewrecord_repostory = DSViewRecordRepository() + self.fileviewrecord_repository = FileViewRecordRepository() + + def get_synchronized(self, current_user_id: int) -> DataSet: + return self.repository.get_synchronized(current_user_id) + + def get_unsynchronized(self, current_user_id: int) -> DataSet: + return self.repository.get_unsynchronized(current_user_id) + + def latest_synchronized(self): + return self.repository.latest_synchronized() + + def filter(self, query="", sorting="newest", publication_type="any", tags=[], **kwargs): + return self.repository.filter(query, sorting, publication_type, tags, **kwargs) + + def count_feature_models(self): + return self.feature_model_service.count() + + def count_authors(self) -> int: + return self.author_repository.count() + + def count_dsmetadata(self) -> int: + return self.dsmetadata_repository.count() + + def total_dataset_downloads(self) -> int: + return self.dsdownloadrecord_repository.total_dataset_downloads() + + def total_feature_model_downloads(self) -> int: + return self.filedownloadrecord_repository.total_feature_model_downloads() + + def total_dataset_views(self) -> int: + return self.dsviewrecord_repostory.total_dataset_views() + + def total_feature_model_views(self) -> int: + return self.fileviewrecord_repository.total_feature_model_views() + + +class FeatureModelService(BaseService): + def __init__(self): + super().__init__(FeatureModelRepository()) + + +class AuthorService(BaseService): + def __init__(self): + super().__init__(AuthorRepository()) + + +class DSDownloadRecordService(BaseService): + def __init__(self): + super().__init__(DSDownloadRecordRepository()) + + +class DSMetaDataService(BaseService): + def __init__(self): + super().__init__(DSMetaDataRepository()) + + def update(self, id, **kwargs): + return self.repository.update(id, **kwargs) + + def filter_by_doi(self, doi: str) -> Optional[DSMetaData]: + return self.repository.filter_by_doi(doi) + + +class DSViewRecordService(BaseService): + def __init__(self): + super().__init__(DSViewRecordRepository()) + + +class FMMetaDataService(BaseService): + def __init__(self): + super().__init__(FMMetaDataRepository()) + + +class FileService(BaseService): + def __init__(self): + super().__init__(FileRepository()) + + +class FileDownloadRecordService(BaseService): + def __init__(self): + super().__init__(FileDownloadRecordRepository()) diff --git a/app/blueprints/dataset/templates/dataset/upload_dataset.html b/app/blueprints/dataset/templates/dataset/upload_dataset.html index a38399087..23313ba6e 100644 --- a/app/blueprints/dataset/templates/dataset/upload_dataset.html +++ b/app/blueprints/dataset/templates/dataset/upload_dataset.html @@ -779,4 +779,4 @@

{% endblock %} -{% endblock %} \ No newline at end of file +{% endblock %} diff --git a/app/blueprints/explore/routes.py b/app/blueprints/explore/routes.py index 34a5944c5..fb38416e9 100644 --- a/app/blueprints/explore/routes.py +++ b/app/blueprints/explore/routes.py @@ -1,13 +1,8 @@ -import re - -import unidecode - from flask import render_template, request, jsonify -from sqlalchemy import or_, any_ from app.blueprints.explore import explore_bp from app.blueprints.explore.forms import ExploreForm -from app.blueprints.dataset.models import DataSet, DSMetaData, Author, FeatureModel, FMMetaData, PublicationType +from app.blueprints.dataset.services import DataSetService @explore_bp.route('/explore', methods=['GET', 'POST']) @@ -16,62 +11,8 @@ def index(): query = request.args.get('query', '') form = ExploreForm() return render_template('explore/index.html', form=form, query=query) - elif request.method == 'POST': - criteria = request.get_json() - query = criteria.get('query', '') - order = criteria.get('sorting', 'newest') - publication_type = criteria.get('publication_type', 'any') - tags = criteria.get('tags', []) - - # Normalize and remove unwanted characters - normalized_query = unidecode.unidecode(query).lower() - cleaned_query = re.sub(r'[,.":\'()\[\]^;!¡¿?]', '', normalized_query) - query_words = cleaned_query.split() - - filters = [] - for word in query_words: - filters.append(DSMetaData.title.ilike(f'%{word}%')) - filters.append(DSMetaData.description.ilike(f'%{word}%')) - filters.append(Author.name.ilike(f'%{word}%')) - filters.append(Author.affiliation.ilike(f'%{word}%')) - filters.append(Author.orcid.ilike(f'%{word}%')) - filters.append(FMMetaData.uvl_filename.ilike(f'%{word}%')) - filters.append(FMMetaData.title.ilike(f'%{word}%')) - filters.append(FMMetaData.description.ilike(f'%{word}%')) - filters.append(FMMetaData.publication_type.ilike(f'%{word}%')) - filters.append(FMMetaData.publication_doi.ilike(f'%{word}%')) - filters.append(FMMetaData.tags.ilike(f'%{word}%')) - filters.append(DSMetaData.tags.ilike(f'%{word}%')) - - datasets = DataSet.query \ - .join(DSMetaData) \ - .join(Author) \ - .join(FeatureModel) \ - .join(FMMetaData) - - if publication_type != 'any': - matching_type = None - for member in PublicationType: - if member.value.lower() == publication_type: - matching_type = member - break - if matching_type is not None: - datasets = datasets.filter(DSMetaData.publication_type == matching_type.name) - - datasets = datasets.filter(or_(*filters)) - - if tags: - datasets = datasets.filter(DSMetaData.tags.ilike(any_(f'%{tag}%' for tag in tags))) - - # Order by created_at - if order == 'oldest': - datasets = datasets.order_by(DataSet.created_at.asc()) - else: - datasets = datasets.order_by(DataSet.created_at.desc()) - - datasets = datasets.all() - - dataset_dicts = [dataset.to_dict() for dataset in datasets] - - return jsonify(dataset_dicts) + if request.method == 'POST': + criteria = request.get_json() + datasets = DataSetService().filter(**criteria) + return jsonify([dataset.to_dict() for dataset in datasets]) diff --git a/app/blueprints/profile/routes.py b/app/blueprints/profile/routes.py index 74b7fddd8..cb46e9d89 100644 --- a/app/blueprints/profile/routes.py +++ b/app/blueprints/profile/routes.py @@ -1,26 +1,29 @@ -from flask import request, render_template +from app.blueprints.dataset.models import DataSet +from flask import render_template, redirect, url_for, request from flask_login import login_required, current_user +from app import get_authenticated_user_profile, db from app.blueprints.profile import profile_bp from app.blueprints.profile.forms import UserProfileForm -from app.blueprints.dataset.models import DataSet - -from app import get_authenticated_user_profile, db from app.blueprints.profile.services import UserProfileService -@profile_bp.route('/profile/edit', methods=['GET', 'POST']) +@profile_bp.route("/profile/edit", methods=["GET", "POST"]) @login_required def edit_profile(): - form = UserProfileForm() - if request.method == 'POST': + profile = get_authenticated_user_profile() + if not profile: + return redirect(url_for("public.index")) + form = UserProfileForm() + if request.method == "POST": service = UserProfileService() - result, errors = service.update_profile(get_authenticated_user_profile().id, form) - return service.handle_service_response(result, errors, 'profile.edit_profile', 'Profile updated successfully', - 'profile/edit.html', form) + result, errors = service.update_profile(profile.id, form) + return service.handle_service_response( + result, errors, "profile.edit_profile", "Profile updated successfully", "profile/edit.html", form + ) - return render_template('profile/edit.html', form=form) + return render_template("profile/edit.html", form=form) @profile_bp.route('/profile/summary') diff --git a/app/blueprints/profile/services.py b/app/blueprints/profile/services.py index b3ff185e4..4dbf9cbce 100644 --- a/app/blueprints/profile/services.py +++ b/app/blueprints/profile/services.py @@ -10,5 +10,5 @@ def update_profile(self, user_profile_id, form): if form.validate(): updated_instance = self.update(user_profile_id, **form.data) return updated_instance, None - else: - return None, form.errors + + return None, form.errors diff --git a/app/blueprints/profile/tests/test_unit.py b/app/blueprints/profile/tests/test_unit.py index 7f6bf9c00..4215d89a7 100644 --- a/app/blueprints/profile/tests/test_unit.py +++ b/app/blueprints/profile/tests/test_unit.py @@ -1,18 +1,25 @@ import pytest +from app import db from app.blueprints.conftest import login, logout +from app.blueprints.auth.models import User +from app.blueprints.profile.models import UserProfile -@pytest.fixture(scope='module') +@pytest.fixture(scope="module") def test_client(test_client): """ Extends the test_client fixture to add additional specific data for module testing. for module testing (por example, new users) """ with test_client.application.app_context(): - # Add HERE new elements to the database that you want to exist in the test context. - # DO NOT FORGET to use db.session.add() and db.session.commit() to save the data. - pass + user_test = User(email='user@example.com', password='test1234') + db.session.add(user_test) + db.session.commit() + + profile = UserProfile(user_id=user_test.id, name="Name", surname="Surname") + db.session.add(profile) + db.session.commit() yield test_client @@ -21,10 +28,10 @@ def test_edit_profile_page_get(test_client): """ Tests access to the profile editing page via a GET request. """ - login_response = login(test_client, 'test@example.com', 'test1234') + login_response = login(test_client, "user@example.com", "test1234") assert login_response.status_code == 200, "Login was unsuccessful." - response = test_client.get('/profile/edit') + response = test_client.get("/profile/edit") assert response.status_code == 200, "The profile editing page could not be accessed." assert b"Edit profile" in response.data, "The expected content is not present on the page" diff --git a/app/blueprints/public/routes.py b/app/blueprints/public/routes.py index df98f8330..5b27b6a65 100644 --- a/app/blueprints/public/routes.py +++ b/app/blueprints/public/routes.py @@ -1,46 +1,33 @@ import logging -from flask_login import login_required - -import app - from flask import render_template + from app.blueprints.public import public_bp -from ..dataset.models import DataSet, DSMetaData, DSDownloadRecord, DSViewRecord, FileDownloadRecord, FileViewRecord +from app.blueprints.dataset.services import DataSetService logger = logging.getLogger(__name__) @public_bp.route("/") def index(): - logger.info('Access index') - - latest_datasets = DataSet.query.join(DSMetaData).filter( - DSMetaData.dataset_doi.isnot(None) - ).order_by(DataSet.created_at.desc()).limit(5).all() - - datasets_counter = app.datasets_counter() - feature_models_counter = app.feature_models_counter() - - # Downloads - total_dataset_downloads = DSDownloadRecord.query.count() - total_feature_model_downloads = FileDownloadRecord.query.count() - - # Views - total_dataset_views = DSViewRecord.query.count() - total_feature_model_views = FileViewRecord.query.count() - - return render_template("public/index.html", - datasets=latest_datasets, - datasets_counter=datasets_counter, - feature_models_counter=feature_models_counter, - total_dataset_downloads=total_dataset_downloads, - total_feature_model_downloads=total_feature_model_downloads, - total_dataset_views=total_dataset_views, - total_feature_model_views=total_feature_model_views) - - -@public_bp.route('/secret') -@login_required -def secret(): - return "Esto es secreto!" + logger.info("Access index") + dataset_service = DataSetService() + + # Statistics: total downloads + total_dataset_downloads = dataset_service.total_dataset_downloads() + total_feature_model_downloads = dataset_service.total_feature_model_downloads() + + # Statistics: total views + total_dataset_views = dataset_service.total_dataset_views() + total_feature_model_views = dataset_service.total_feature_model_views() + + return render_template( + "public/index.html", + datasets=dataset_service.latest_synchronized(), + datasets_counter=dataset_service.count(), + feature_models_counter=dataset_service.count_feature_models(), + total_dataset_downloads=total_dataset_downloads, + total_feature_model_downloads=total_feature_model_downloads, + total_dataset_views=total_dataset_views, + total_feature_model_views=total_feature_model_views + ) diff --git a/app/blueprints/pytest.ini b/app/blueprints/pytest.ini index b0e5a945f..c24fe5bb9 100644 --- a/app/blueprints/pytest.ini +++ b/app/blueprints/pytest.ini @@ -1,3 +1,3 @@ [pytest] filterwarnings = - ignore::DeprecationWarning \ No newline at end of file + ignore::DeprecationWarning diff --git a/app/blueprints/zenodo/routes.py b/app/blueprints/zenodo/routes.py index e3a4ba8e8..1506d5fb7 100644 --- a/app/blueprints/zenodo/routes.py +++ b/app/blueprints/zenodo/routes.py @@ -1,7 +1,15 @@ from flask import render_template + from app.blueprints.zenodo import zenodo_bp +from app.blueprints.zenodo.services import ZenodoService @zenodo_bp.route('/zenodo', methods=['GET']) def index(): return render_template('zenodo/index.html') + + +@zenodo_bp.route('/zenodo/test', methods=['GET']) +def zenodo_test() -> dict: + service = ZenodoService() + return service.test_full_connection() diff --git a/app/blueprints/zenodo/services.py b/app/blueprints/zenodo/services.py index 3e3febbc3..3c8d4ca34 100644 --- a/app/blueprints/zenodo/services.py +++ b/app/blueprints/zenodo/services.py @@ -1,8 +1,4 @@ -from app.blueprints.zenodo.repositories import ZenodoRepository -from core.services.BaseService import BaseService - import os - import requests from dotenv import load_dotenv @@ -11,215 +7,210 @@ import app from app.blueprints.dataset.models import DataSet, FeatureModel +from app.blueprints.zenodo.repositories import ZenodoRepository +from core.services.BaseService import BaseService -load_dotenv() - -ZENODO_API_URL = 'https://sandbox.zenodo.org/api/deposit/depositions' -ZENODO_ACCESS_TOKEN = os.getenv('ZENODO_ACCESS_TOKEN') - - -def test_zenodo_connection() -> bool: - """ - Test the connection with Zenodo. - - Returns: - bool: True if the connection is successful, False otherwise. - """ - headers = {"Content-Type": "application/json"} - params = {'access_token': ZENODO_ACCESS_TOKEN} - response = requests.get(ZENODO_API_URL, params=params, headers=headers) - return response.status_code == 200 - - -def test_full_zenodo_connection() -> Response: - """ - Test the connection with Zenodo by creating a deposition, uploading an empty test file, and deleting the deposition. - - Returns: - bool: True if the connection, upload, and deletion are successful, False otherwise. - """ - success = True +load_dotenv() - # Create an empty file - file_path = os.path.join(current_app.root_path, "test_file.txt") - with open(file_path, 'w'): - pass - messages = [] # List to store messages +class ZenodoService(BaseService): + ZENODO_API_URL = os.getenv("ZENODO_API_URL", "https://sandbox.zenodo.org/api/deposit/depositions") + ZENODO_ACCESS_TOKEN = os.getenv("ZENODO_ACCESS_TOKEN") - # Step 1: Create a deposition on Zenodo - headers = {"Content-Type": "application/json"} - data = { - "metadata": { - "title": "Test Deposition", - "upload_type": "dataset", - "description": "This is a test deposition created via Zenodo API", - "creators": [{"name": "John Doe"}] - } - } - - params = {'access_token': ZENODO_ACCESS_TOKEN} - response = requests.post(ZENODO_API_URL, json=data, params=params, headers=headers) - - if response.status_code != 201: - messages.append(f"Failed to create test deposition on Zenodo. Response code: {response.status_code}") - success = False - - deposition_id = response.json()["id"] - - # Step 2: Upload an empty file to the deposition - data = {'name': "test_file.txt"} - file_path = os.path.join(current_app.root_path, "test_file.txt") - files = {'file': open(file_path, 'rb')} - publish_url = f'{ZENODO_API_URL}/{deposition_id}/files' - response = requests.post(publish_url, params=params, data=data, files=files) - - if response.status_code != 201: - messages.append(f"Failed to upload test file to Zenodo. Response code: {response.status_code}") - success = False - - # Step 3: Delete the deposition - response = requests.delete(f"{ZENODO_API_URL}/{deposition_id}", params=params) - - if os.path.exists(file_path): - os.remove(file_path) - - return jsonify({"success": success, "messages": messages}) - - -def get_all_depositions() -> dict: - """ - Get all depositions from Zenodo. - - Returns: - dict: The response in JSON format with the depositions. - """ - headers = {"Content-Type": "application/json"} - params = {'access_token': ZENODO_ACCESS_TOKEN} - response = requests.get(ZENODO_API_URL, params=params, headers=headers) - if response.status_code != 200: - raise Exception('Failed to get depositions') - return response.json() - - -def zenodo_create_new_deposition(dataset: DataSet) -> dict: - """ - Create a new deposition in Zenodo. - - Args: - dataset (DataSet): The DataSet object containing the metadata of the deposition. - - Returns: - dict: The response in JSON format with the details of the created deposition. - """ - metadata = { - 'title': dataset.ds_meta_data.title, - 'upload_type': 'dataset' if dataset.ds_meta_data.publication_type.value == "none" else 'publication', - 'publication_type': dataset.ds_meta_data.publication_type.value - if dataset.ds_meta_data.publication_type.value != "none" else None, - 'description': dataset.ds_meta_data.description, - 'creators': [{ - 'name': author.name, - **({'affiliation': author.affiliation} if author.affiliation else {}), - **({'orcid': author.orcid} if author.orcid else {}) - } for author in dataset.ds_meta_data.authors], - 'keywords': ["uvlhub"] if not dataset.ds_meta_data.tags else dataset.ds_meta_data.tags.split(", ") + ["uvlhub"], - 'access_right': 'open', - 'license': 'CC-BY-4.0' - } - - data = { - 'metadata': metadata - } - - headers = {"Content-Type": "application/json"} - params = {'access_token': ZENODO_ACCESS_TOKEN} - response = requests.post(ZENODO_API_URL, params=params, json=data, headers=headers) - if response.status_code != 201: - error_message = f'Failed to create deposition. Error details: {response.json()}' - raise Exception(error_message) - return response.json() - - -def zenodo_upload_file(deposition_id: int, feature_model: FeatureModel, user=None) -> dict: - """ - Upload a file to a deposition in Zenodo. - - Args: - deposition_id (int): The ID of the deposition in Zenodo. - feature_model (FeatureModel): The FeatureModel object representing the feature model. - user (FeatureModel): The User object representing the file owner. - - Returns: - dict: The response in JSON format with the details of the uploaded file. - """ - uvl_filename = feature_model.fm_meta_data.uvl_filename - data = {'name': uvl_filename} - user_id = current_user.id if user is None else user.id - file_path = os.path.join(app.upload_folder_name(), 'temp', str(user_id), uvl_filename) - files = {'file': open(file_path, 'rb')} - - publish_url = f'{ZENODO_API_URL}/{deposition_id}/files' - params = {'access_token': ZENODO_ACCESS_TOKEN} - response = requests.post(publish_url, params=params, data=data, files=files) - if response.status_code != 201: - error_message = f'Failed to upload files. Error details: {response.json()}' - raise Exception(error_message) - return response.json() - - -def zenodo_publish_deposition(deposition_id: int) -> dict: - """ - Publish a deposition in Zenodo. - - Args: - deposition_id (int): The ID of the deposition in Zenodo. - - Returns: - dict: The response in JSON format with the details of the published deposition. - """ - headers = {"Content-Type": "application/json"} - publish_url = f'{ZENODO_API_URL}/{deposition_id}/actions/publish' - params = {'access_token': ZENODO_ACCESS_TOKEN} - response = requests.post(publish_url, params=params, headers=headers) - if response.status_code != 202: - raise Exception('Failed to publish deposition') - return response.json() - - -def zenodo_get_deposition(deposition_id: int) -> dict: - """ - Get a deposition from Zenodo. - - Args: - deposition_id (int): The ID of the deposition in Zenodo. - - Returns: - dict: The response in JSON format with the details of the deposition. - """ - headers = {"Content-Type": "application/json"} - deposition_url = f'{ZENODO_API_URL}/{deposition_id}' - params = {'access_token': ZENODO_ACCESS_TOKEN} - response = requests.get(deposition_url, params=params, headers=headers) - if response.status_code != 200: - raise Exception('Failed to get deposition') - return response.json() - - -def zenodo_get_doi(deposition_id: int) -> str: - """ - Get the DOI of a deposition from Zenodo. - - Args: - deposition_id (int): The ID of the deposition in Zenodo. - - Returns: - str: The DOI of the deposition. - """ - return zenodo_get_deposition(deposition_id).get('doi') - - -class Zenodo(BaseService): def __init__(self): super().__init__(ZenodoRepository()) + self.headers = {"Content-Type": "application/json"} + self.params = {"access_token": self.ZENODO_ACCESS_TOKEN} + + def test_connection(self) -> bool: + """ + Test the connection with Zenodo. + + Returns: + bool: True if the connection is successful, False otherwise. + """ + response = requests.get(self.ZENODO_API_URL, params=self.params, headers=self.headers) + return response.status_code == 200 + + def test_full_connection(self) -> Response: + """ + Test the connection with Zenodo by creating a deposition, uploading an empty test file, and deleting the + deposition. + + Returns: + bool: True if the connection, upload, and deletion are successful, False otherwise. + """ + + success = True + + # Create an empty file + file_path = os.path.join(current_app.root_path, "test_file.txt") + with open(file_path, "w"): + pass + + messages = [] # List to store messages + + # Step 1: Create a deposition on Zenodo + data = { + "metadata": { + "title": "Test Deposition", + "upload_type": "dataset", + "description": "This is a test deposition created via Zenodo API", + "creators": [{"name": "John Doe"}], + } + } + + response = requests.post(self.ZENODO_API_URL, json=data, params=self.params, headers=self.headers) + + if response.status_code != 201: + return jsonify( + { + "success": False, + "messages": f"Failed to create test deposition on Zenodo. Response code: {response.status_code}", + } + ) + + deposition_id = response.json()["id"] + + # Step 2: Upload an empty file to the deposition + data = {"name": "test_file.txt"} + file_path = os.path.join(current_app.root_path, "test_file.txt") + files = {"file": open(file_path, "rb")} + publish_url = f"{self.ZENODO_API_URL}/{deposition_id}/files" + response = requests.post(publish_url, params=self.params, data=data, files=files) + + if response.status_code != 201: + messages.append(f"Failed to upload test file to Zenodo. Response code: {response.status_code}") + success = False + + # Step 3: Delete the deposition + response = requests.delete(f"{self.ZENODO_API_URL}/{deposition_id}", params=self.params) + + if os.path.exists(file_path): + os.remove(file_path) + + return jsonify({"success": success, "messages": messages}) + + def get_all_depositions(self) -> dict: + """ + Get all depositions from Zenodo. + + Returns: + dict: The response in JSON format with the depositions. + """ + response = requests.get(self.ZENODO_API_URL, params=self.params, headers=self.headers) + if response.status_code != 200: + raise Exception("Failed to get depositions") + return response.json() + + def create_new_deposition(self, dataset: DataSet) -> dict: + """ + Create a new deposition in Zenodo. + + Args: + dataset (DataSet): The DataSet object containing the metadata of the deposition. + + Returns: + dict: The response in JSON format with the details of the created deposition. + """ + metadata = { + "title": dataset.ds_meta_data.title, + "upload_type": "dataset" if dataset.ds_meta_data.publication_type.value == "none" else "publication", + "publication_type": ( + dataset.ds_meta_data.publication_type.value + if dataset.ds_meta_data.publication_type.value != "none" + else None + ), + "description": dataset.ds_meta_data.description, + "creators": [ + { + "name": author.name, + **({"affiliation": author.affiliation} if author.affiliation else {}), + **({"orcid": author.orcid} if author.orcid else {}), + } + for author in dataset.ds_meta_data.authors + ], + "keywords": ( + ["uvlhub"] if not dataset.ds_meta_data.tags else dataset.ds_meta_data.tags.split(", ") + ["uvlhub"] + ), + "access_right": "open", + "license": "CC-BY-4.0", + } + + data = {"metadata": metadata} + + response = requests.post(self.ZENODO_API_URL, params=self.params, json=data, headers=self.headers) + if response.status_code != 201: + error_message = f"Failed to create deposition. Error details: {response.json()}" + raise Exception(error_message) + return response.json() + + def upload_file(self, deposition_id: int, feature_model: FeatureModel, user=None) -> dict: + """ + Upload a file to a deposition in Zenodo. + + Args: + deposition_id (int): The ID of the deposition in Zenodo. + feature_model (FeatureModel): The FeatureModel object representing the feature model. + user (FeatureModel): The User object representing the file owner. + + Returns: + dict: The response in JSON format with the details of the uploaded file. + """ + uvl_filename = feature_model.fm_meta_data.uvl_filename + data = {"name": uvl_filename} + user_id = current_user.id if user is None else user.id + file_path = os.path.join(app.upload_folder_name(), "temp", str(user_id), uvl_filename) + files = {"file": open(file_path, "rb")} + + publish_url = f"{self.ZENODO_API_URL}/{deposition_id}/files" + response = requests.post(publish_url, params=self.params, data=data, files=files) + if response.status_code != 201: + error_message = f"Failed to upload files. Error details: {response.json()}" + raise Exception(error_message) + return response.json() + + def publish_deposition(self, deposition_id: int) -> dict: + """ + Publish a deposition in Zenodo. + + Args: + deposition_id (int): The ID of the deposition in Zenodo. + + Returns: + dict: The response in JSON format with the details of the published deposition. + """ + publish_url = f"{self.ZENODO_API_URL}/{deposition_id}/actions/publish" + response = requests.post(publish_url, params=self.params, headers=self.headers) + if response.status_code != 202: + raise Exception("Failed to publish deposition") + return response.json() + + def get_deposition(self, deposition_id: int) -> dict: + """ + Get a deposition from Zenodo. + + Args: + deposition_id (int): The ID of the deposition in Zenodo. + + Returns: + dict: The response in JSON format with the details of the deposition. + """ + deposition_url = f"{self.ZENODO_API_URL}/{deposition_id}" + response = requests.get(deposition_url, params=self.params, headers=self.headers) + if response.status_code != 200: + raise Exception("Failed to get deposition") + return response.json() + + def get_doi(self, deposition_id: int) -> str: + """ + Get the DOI of a deposition from Zenodo. + + Args: + deposition_id (int): The ID of the deposition in Zenodo. + + Returns: + str: The DOI of the deposition. + """ + return self.get_deposition(deposition_id).get("doi") diff --git a/app/templates/base_template.html b/app/templates/base_template.html index 418b25dad..11d8d83fa 100644 --- a/app/templates/base_template.html +++ b/app/templates/base_template.html @@ -235,4 +235,4 @@ - + \ No newline at end of file diff --git a/core/repositories/BaseRepository.py b/core/repositories/BaseRepository.py index d46428380..f00892e40 100644 --- a/core/repositories/BaseRepository.py +++ b/core/repositories/BaseRepository.py @@ -1,37 +1,47 @@ -from typing import TypeVar, Generic, Optional - -import app - -T = TypeVar('T') - - -class BaseRepository(Generic[T]): - def __init__(self, model: T): - self.model = model - - def create(self, **kwargs) -> T: - instance: T = self.model(**kwargs) - app.db.session.add(instance) - app.db.session.commit() - return instance - - def get_by_id(self, id: int) -> Optional[T]: - instance: Optional[T] = self.model.query.get(id) - return instance - - def update(self, id: int, **kwargs) -> Optional[T]: - instance: Optional[T] = self.get_by_id(id) - if instance: - for key, value in kwargs.items(): - setattr(instance, key, value) - app.db.session.commit() - return instance - return None - - def delete(self, id: int) -> bool: - instance: Optional[T] = self.get_by_id(id) - if instance: - app.db.session.delete(instance) - app.db.session.commit() - return True - return False +from typing import Generic, NoReturn, Optional, TypeVar, Union + +import app + +T = TypeVar('T') + + +class BaseRepository(Generic[T]): + def __init__(self, model: T): + self.model = model + self.session = app.db.session + + def create(self, commit: bool = True, **kwargs) -> T: + instance: T = self.model(**kwargs) + self.session.add(instance) + if commit: + self.session.commit() + else: + self.session.flush() + return instance + + def get_by_id(self, id: int) -> Optional[T]: + instance: Optional[T] = self.model.query.get(id) + return instance + + def get_or_404(self, id: int) -> Union[T, NoReturn]: + return self.model.query.get_or_404(id) + + def update(self, id: int, **kwargs) -> Optional[T]: + instance: Optional[T] = self.get_by_id(id) + if instance: + for key, value in kwargs.items(): + setattr(instance, key, value) + self.session.commit() + return instance + return None + + def delete(self, id: int) -> bool: + instance: Optional[T] = self.get_by_id(id) + if instance: + self.session.delete(instance) + self.session.commit() + return True + return False + + def count(self) -> int: + return self.model.query.count() diff --git a/core/services/BaseService.py b/core/services/BaseService.py index 3ff8d6472..942dca6b8 100644 --- a/core/services/BaseService.py +++ b/core/services/BaseService.py @@ -1,28 +1,34 @@ -from flask import flash, redirect, url_for, render_template - - -class BaseService: - def __init__(self, repository): - self.repository = repository - - def create(self, **kwargs): - return self.repository.create(**kwargs) - - def get_by_id(self, id): - return self.repository.get_by_id(id) - - def update(self, id, **kwargs): - return self.repository.update(id, **kwargs) - - def delete(self, id): - return self.repository.delete(id) - - def handle_service_response(self, result, errors, success_url_redirect, success_msg, error_template, form): - if result: - flash(success_msg, 'success') - return redirect(url_for(success_url_redirect)) - else: - for error_field, error_messages in errors.items(): - for error_message in error_messages: - flash(f'{error_field}: {error_message}', 'error') - return render_template(error_template, form=form) +from flask import flash, redirect, url_for, render_template + + +class BaseService: + def __init__(self, repository): + self.repository = repository + + def create(self, **kwargs): + return self.repository.create(**kwargs) + + def count(self) -> int: + return self.repository.count() + + def get_by_id(self, id): + return self.repository.get_by_id(id) + + def get_or_404(self, id): + return self.repository.get_or_404(id) + + def update(self, id, **kwargs): + return self.repository.update(id, **kwargs) + + def delete(self, id): + return self.repository.delete(id) + + def handle_service_response(self, result, errors, success_url_redirect, success_msg, error_template, form): + if result: + flash(success_msg, 'success') + return redirect(url_for(success_url_redirect)) + else: + for error_field, error_messages in errors.items(): + for error_message in error_messages: + flash(f'{error_field}: {error_message}', 'error') + return render_template(error_template, form=form) diff --git a/migrations/versions/2645480ae393_.py b/migrations/versions/2645480ae393_.py deleted file mode 100644 index 5d5b91610..000000000 --- a/migrations/versions/2645480ae393_.py +++ /dev/null @@ -1,31 +0,0 @@ -"""empty message - -Revision ID: 2645480ae393 -Revises: -Create Date: 2024-06-25 10:29:57.644851 - -""" -from alembic import op -import sqlalchemy as sa - - -# revision identifiers, used by Alembic. -revision = '2645480ae393' -down_revision = None -branch_labels = None -depends_on = None - - -def upgrade(): - # ### commands auto generated by Alembic - please adjust! ### - op.create_table('webhook', - sa.Column('id', sa.Integer(), nullable=False), - sa.PrimaryKeyConstraint('id') - ) - # ### end Alembic commands ### - - -def downgrade(): - # ### commands auto generated by Alembic - please adjust! ### - op.drop_table('webhook') - # ### end Alembic commands ### diff --git a/migrations/versions/97d142f05ca0_.py b/migrations/versions/97d142f05ca0_.py new file mode 100644 index 000000000..d314dcde2 --- /dev/null +++ b/migrations/versions/97d142f05ca0_.py @@ -0,0 +1,179 @@ +"""empty message + +Revision ID: 97d142f05ca0 +Revises: +Create Date: 2024-07-03 07:47:24.228683 + +""" +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision = '97d142f05ca0' +down_revision = None +branch_labels = None +depends_on = None + + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.create_table('ds_metrics', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('number_of_models', sa.String(length=120), nullable=True), + sa.Column('number_of_features', sa.String(length=120), nullable=True), + sa.PrimaryKeyConstraint('id') + ) + op.create_table('fm_metrics', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('solver', sa.Text(), nullable=True), + sa.Column('not_solver', sa.Text(), nullable=True), + sa.PrimaryKeyConstraint('id') + ) + op.create_table('user', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('email', sa.String(length=256), nullable=False), + sa.Column('password', sa.String(length=256), nullable=False), + sa.Column('created_at', sa.DateTime(), nullable=False), + sa.PrimaryKeyConstraint('id'), + sa.UniqueConstraint('email') + ) + op.create_table('zenodo', + sa.Column('id', sa.Integer(), nullable=False), + sa.PrimaryKeyConstraint('id') + ) + op.create_table('ds_meta_data', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('deposition_id', sa.Integer(), nullable=True), + sa.Column('title', sa.String(length=120), nullable=False), + sa.Column('description', sa.Text(), nullable=False), + sa.Column('publication_type', sa.Enum('NONE', 'ANNOTATION_COLLECTION', 'BOOK', 'BOOK_SECTION', 'CONFERENCE_PAPER', 'DATA_MANAGEMENT_PLAN', 'JOURNAL_ARTICLE', 'PATENT', 'PREPRINT', 'PROJECT_DELIVERABLE', 'PROJECT_MILESTONE', 'PROPOSAL', 'REPORT', 'SOFTWARE_DOCUMENTATION', 'TAXONOMIC_TREATMENT', 'TECHNICAL_NOTE', 'THESIS', 'WORKING_PAPER', 'OTHER', name='publicationtype'), nullable=False), + sa.Column('publication_doi', sa.String(length=120), nullable=True), + sa.Column('dataset_doi', sa.String(length=120), nullable=True), + sa.Column('tags', sa.String(length=120), nullable=True), + sa.Column('ds_metrics_id', sa.Integer(), nullable=True), + sa.ForeignKeyConstraint(['ds_metrics_id'], ['ds_metrics.id'], ), + sa.PrimaryKeyConstraint('id') + ) + op.create_table('fm_meta_data', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('uvl_filename', sa.String(length=120), nullable=False), + sa.Column('title', sa.String(length=120), nullable=False), + sa.Column('description', sa.Text(), nullable=False), + sa.Column('publication_type', sa.Enum('NONE', 'ANNOTATION_COLLECTION', 'BOOK', 'BOOK_SECTION', 'CONFERENCE_PAPER', 'DATA_MANAGEMENT_PLAN', 'JOURNAL_ARTICLE', 'PATENT', 'PREPRINT', 'PROJECT_DELIVERABLE', 'PROJECT_MILESTONE', 'PROPOSAL', 'REPORT', 'SOFTWARE_DOCUMENTATION', 'TAXONOMIC_TREATMENT', 'TECHNICAL_NOTE', 'THESIS', 'WORKING_PAPER', 'OTHER', name='publicationtype'), nullable=False), + sa.Column('publication_doi', sa.String(length=120), nullable=True), + sa.Column('tags', sa.String(length=120), nullable=True), + sa.Column('uvl_version', sa.String(length=120), nullable=True), + sa.Column('fm_metrics_id', sa.Integer(), nullable=True), + sa.ForeignKeyConstraint(['fm_metrics_id'], ['fm_metrics.id'], ), + sa.PrimaryKeyConstraint('id') + ) + op.create_table('user_profile', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('user_id', sa.Integer(), nullable=False), + sa.Column('orcid', sa.String(length=19), nullable=True), + sa.Column('affiliation', sa.String(length=100), nullable=True), + sa.Column('name', sa.String(length=100), nullable=False), + sa.Column('surname', sa.String(length=100), nullable=False), + sa.ForeignKeyConstraint(['user_id'], ['user.id'], ), + sa.PrimaryKeyConstraint('id'), + sa.UniqueConstraint('user_id') + ) + op.create_table('author', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('name', sa.String(length=120), nullable=False), + sa.Column('affiliation', sa.String(length=120), nullable=True), + sa.Column('orcid', sa.String(length=120), nullable=True), + sa.Column('ds_meta_data_id', sa.Integer(), nullable=True), + sa.Column('fm_meta_data_id', sa.Integer(), nullable=True), + sa.ForeignKeyConstraint(['ds_meta_data_id'], ['ds_meta_data.id'], ), + sa.ForeignKeyConstraint(['fm_meta_data_id'], ['fm_meta_data.id'], ), + sa.PrimaryKeyConstraint('id') + ) + op.create_table('data_set', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('user_id', sa.Integer(), nullable=False), + sa.Column('ds_meta_data_id', sa.Integer(), nullable=False), + sa.Column('created_at', sa.DateTime(), nullable=False), + sa.ForeignKeyConstraint(['ds_meta_data_id'], ['ds_meta_data.id'], ), + sa.ForeignKeyConstraint(['user_id'], ['user.id'], ), + sa.PrimaryKeyConstraint('id') + ) + op.create_table('ds_download_record', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('user_id', sa.Integer(), nullable=True), + sa.Column('dataset_id', sa.Integer(), nullable=True), + sa.Column('download_date', sa.DateTime(), nullable=False), + sa.Column('download_cookie', sa.String(length=36), nullable=False), + sa.ForeignKeyConstraint(['dataset_id'], ['data_set.id'], ), + sa.ForeignKeyConstraint(['user_id'], ['user.id'], ), + sa.PrimaryKeyConstraint('id') + ) + op.create_table('ds_view_record', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('user_id', sa.Integer(), nullable=True), + sa.Column('dataset_id', sa.Integer(), nullable=True), + sa.Column('view_date', sa.DateTime(), nullable=False), + sa.Column('view_cookie', sa.String(length=36), nullable=False), + sa.ForeignKeyConstraint(['dataset_id'], ['data_set.id'], ), + sa.ForeignKeyConstraint(['user_id'], ['user.id'], ), + sa.PrimaryKeyConstraint('id') + ) + op.create_table('feature_model', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('data_set_id', sa.Integer(), nullable=False), + sa.Column('fm_meta_data_id', sa.Integer(), nullable=True), + sa.ForeignKeyConstraint(['data_set_id'], ['data_set.id'], ), + sa.ForeignKeyConstraint(['fm_meta_data_id'], ['fm_meta_data.id'], ), + sa.PrimaryKeyConstraint('id') + ) + op.create_table('file', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('name', sa.String(length=120), nullable=False), + sa.Column('checksum', sa.String(length=120), nullable=False), + sa.Column('size', sa.Integer(), nullable=False), + sa.Column('feature_model_id', sa.Integer(), nullable=False), + sa.ForeignKeyConstraint(['feature_model_id'], ['feature_model.id'], ), + sa.PrimaryKeyConstraint('id') + ) + op.create_table('file_download_record', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('user_id', sa.Integer(), nullable=True), + sa.Column('file_id', sa.Integer(), nullable=True), + sa.Column('download_date', sa.DateTime(), nullable=False), + sa.Column('download_cookie', sa.String(length=36), nullable=False), + sa.ForeignKeyConstraint(['file_id'], ['file.id'], ), + sa.ForeignKeyConstraint(['user_id'], ['user.id'], ), + sa.PrimaryKeyConstraint('id') + ) + op.create_table('file_view_record', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('user_id', sa.Integer(), nullable=True), + sa.Column('file_id', sa.Integer(), nullable=False), + sa.Column('view_date', sa.DateTime(), nullable=True), + sa.Column('view_cookie', sa.String(length=36), nullable=True), + sa.ForeignKeyConstraint(['file_id'], ['file.id'], ), + sa.ForeignKeyConstraint(['user_id'], ['user.id'], ), + sa.PrimaryKeyConstraint('id') + ) + # ### end Alembic commands ### + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.drop_table('file_view_record') + op.drop_table('file_download_record') + op.drop_table('file') + op.drop_table('feature_model') + op.drop_table('ds_view_record') + op.drop_table('ds_download_record') + op.drop_table('data_set') + op.drop_table('author') + op.drop_table('user_profile') + op.drop_table('fm_meta_data') + op.drop_table('ds_meta_data') + op.drop_table('zenodo') + op.drop_table('user') + op.drop_table('fm_metrics') + op.drop_table('ds_metrics') + # ### end Alembic commands ### diff --git a/requirements.txt b/requirements.txt index bce6c0dc2..d85c32085 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -alembic==1.13.1 +alembic==1.13.2 aniso8601==9.0.1 attrs==23.2.0 beautifulsoup4==4.12.3 @@ -10,12 +10,12 @@ cffi==1.16.0 charset-normalizer==3.3.2 click==8.1.7 ConfigArgParse==1.7 -coverage==7.5.3 +coverage==7.5.4 cryptography==42.0.8 dnspython==2.6.1 docker==7.1.0 email_validator==2.2.0 -Faker==25.9.1 +Faker==26.0.0 flake8==7.1.0 Flask==3.0.3 Flask-Cors==4.0.1 @@ -37,7 +37,7 @@ iniconfig==2.0.0 itsdangerous==2.2.0 Jinja2==3.1.4 kaitaistruct==0.10 -locust==2.29.0 +locust==2.29.1 Mako==1.3.5 MarkupSafe==2.1.5 mccabe==0.7.0 @@ -63,7 +63,7 @@ pyzmq==26.0.3 requests==2.32.3 selenium==4.22.0 selenium-wire==5.1.0 -setuptools==70.1.0 +setuptools==70.2.0 six==1.16.0 sniffio==1.3.1 sortedcontainers==2.4.0 diff --git a/rosemary/commands/coverage.py b/rosemary/commands/coverage.py index 3d58671f5..6870e8516 100644 --- a/rosemary/commands/coverage.py +++ b/rosemary/commands/coverage.py @@ -19,7 +19,7 @@ def coverage(module_name, html): else: click.echo("Running coverage for all modules...") - coverage_cmd = ['pytest', '--cov=' + test_path, test_path] + coverage_cmd = ['pytest', '--ignore-glob=*selenium*', '--cov=' + test_path, test_path] if html: coverage_cmd.extend(['--cov-report', 'html'])