From 8e81de6b2446a93b40ac0dd84236434a2da0595e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?V=C3=ADctor=20Ram=C3=ADrez=20de=20la=20Corte?= Date: Fri, 31 May 2024 08:32:37 +0200 Subject: [PATCH 1/7] refactor: create ZenodoService, create Dataset repositories and both applied to Dataset routes --- app/blueprints/conftest.py | 1 + app/blueprints/dataset/models.py | 7 +- app/blueprints/dataset/repositories.py | 82 +++ app/blueprints/dataset/routes.py | 777 +++++++++---------------- app/blueprints/zenodo/routes.py | 8 + app/blueprints/zenodo/services.py | 438 +++++++------- core/repositories/BaseRepository.py | 5 +- 7 files changed, 588 insertions(+), 730 deletions(-) create mode 100644 app/blueprints/dataset/repositories.py diff --git a/app/blueprints/conftest.py b/app/blueprints/conftest.py index bd62fa990..3731e5fdf 100644 --- a/app/blueprints/conftest.py +++ b/app/blueprints/conftest.py @@ -1,4 +1,5 @@ import pytest + from app import create_app, db from app.blueprints.auth.models import User diff --git a/app/blueprints/dataset/models.py b/app/blueprints/dataset/models.py index 06e26e7b8..1dfb3bfc7 100644 --- a/app/blueprints/dataset/models.py +++ b/app/blueprints/dataset/models.py @@ -1,12 +1,11 @@ +import os from datetime import datetime +from enum import Enum from flask import request - -from app import db -from enum import Enum from sqlalchemy import Enum as SQLAlchemyEnum -import os +from app import db class PublicationType(Enum): diff --git a/app/blueprints/dataset/repositories.py b/app/blueprints/dataset/repositories.py new file mode 100644 index 000000000..c1d4aa897 --- /dev/null +++ b/app/blueprints/dataset/repositories.py @@ -0,0 +1,82 @@ +from typing import Optional + +from app.blueprints.dataset.models import ( + Author, + DSDownloadRecord, + DSMetaData, + DSViewRecord, + DataSet, + FMMetaData, + FeatureModel, + File, + FileDownloadRecord, +) +from core.repositories.BaseRepository import BaseRepository + + +class AuthorRepository(BaseRepository): + def __init__(self): + super().__init__(Author) + + +class DSDownloadRecordRepository(BaseRepository): + def __init__(self): + super().__init__(DSDownloadRecord) + + +class DSMetaDataRepository(BaseRepository): + def __init__(self): + super().__init__(DSMetaData) + + def filter_by_doi(self, doi: str) -> Optional[DSMetaData]: + return self.model.query.filter_by(dataset_doi=doi).first() + + +class DSViewRecordRepository(BaseRepository): + def __init__(self): + super().__init__(DSViewRecord) + + +class DataSetRepository(BaseRepository): + def __init__(self): + super().__init__(DataSet) + + def get_synchronized(self, current_user_id: int) -> DataSet: + return ( + self.model.query.join(DSMetaData) + .filter( + DataSet.user_id == current_user_id, DSMetaData.dataset_doi.isnot(None) + ) + .order_by(DataSet.created_at.desc()) + .all() + ) + + def get_unsynchronized(self, current_user_id: int) -> DataSet: + return ( + self.model.query.join(DSMetaData) + .filter( + DataSet.user_id == current_user_id, DSMetaData.dataset_doi.is_(None) + ) + .order_by(DataSet.created_at.desc()) + .all() + ) + + +class FMMetaDataRepository(BaseRepository): + def __init__(self): + super().__init__(FMMetaData) + + +class FeatureModelRepository(BaseRepository): + def __init__(self): + super().__init__(FeatureModel) + + +class FileRepository(BaseRepository): + def __init__(self): + super().__init__(File) + + +class FileDownloadRecordRepository(BaseRepository): + def __init__(self): + super().__init__(FileDownloadRecord) diff --git a/app/blueprints/dataset/routes.py b/app/blueprints/dataset/routes.py index 1997fb01f..1cf85675b 100644 --- a/app/blueprints/dataset/routes.py +++ b/app/blueprints/dataset/routes.py @@ -7,70 +7,88 @@ from datetime import datetime from zipfile import ZipFile -from flask import render_template, request, jsonify, send_from_directory, current_app, make_response, abort +from flask import ( + render_template, + request, + jsonify, + send_from_directory, + current_app, + make_response, + abort, +) from flask_login import login_required, current_user import app from app.blueprints.dataset.forms import DataSetForm -from app.blueprints.dataset.models import DataSet, FeatureModel, File, FMMetaData, DSMetaData, Author, \ - PublicationType, DSDownloadRecord, DSViewRecord, FileDownloadRecord +from app.blueprints.dataset.models import DataSet, PublicationType from app.blueprints.dataset import dataset_bp -from app.blueprints.zenodo.services import test_full_zenodo_connection, zenodo_create_new_deposition, \ - zenodo_upload_file, zenodo_publish_deposition, zenodo_get_doi +from app.blueprints.dataset.repositories import ( + AuthorRepository, + DSDownloadRecordRepository, + DSMetaDataRepository, + DSViewRecordRepository, + DataSetRepository, + FMMetaDataRepository, + FeatureModelRepository, + FileRepository, + FileDownloadRecordRepository, +) +from app.blueprints.zenodo.services import ZenodoService -@dataset_bp.route('/zenodo/test', methods=['GET']) -def zenodo_test() -> dict: - return test_full_zenodo_connection() +zenodo_service = ZenodoService() -@dataset_bp.route('/dataset/upload', methods=['GET', 'POST']) +@dataset_bp.route("/dataset/upload", methods=["GET", "POST"]) @login_required def create_dataset(): form = DataSetForm() - if request.method == 'POST': - + if request.method == "POST": try: - # get JSON from frontend - form_data_json = request.form.get('formData') + form_data_json = request.form.get("formData") form_data_dict = json.loads(form_data_json) # get dicts - basic_info_data = form_data_dict["basic_info_form"] uploaded_models_data = form_data_dict["uploaded_models_form"] # create dataset - dataset = create_dataset_in_db(basic_info_data) + dataset = create_dataset_in_db(form_data_dict["basic_info_form"]) # send dataset as deposition to Zenodo - zenodo_response_json = zenodo_create_new_deposition(dataset) - - response_data = json.dumps(zenodo_response_json) - data = json.loads(response_data) - if data.get('conceptrecid'): + try: + zenodo_response_json = zenodo_service.create_new_deposition(dataset) + response_data = json.dumps(zenodo_response_json) + data = json.loads(response_data) + except Exception: + data = {} - deposition_id = data.get('id') + if data.get("conceptrecid"): + deposition_id = data.get("id") # update dataset with deposition id in Zenodo - dataset.ds_meta_data.deposition_id = deposition_id - app.db.session.commit() + DSMetaDataRepository().update( + dataset.ds_meta_data_id, deposition_id=deposition_id + ) # create feature models - feature_models = create_feature_models_in_db(dataset, uploaded_models_data) + feature_models = create_feature_models_in_db( + dataset, uploaded_models_data + ) try: # iterate for each feature model (one feature model = one request to Zenodo for feature_model in feature_models: - zenodo_upload_file(deposition_id, feature_model) + zenodo_service.upload_file(deposition_id, feature_model) # publish deposition - zenodo_publish_deposition(deposition_id) + zenodo_service.publish_deposition(deposition_id) # update DOI - deposition_doi = zenodo_get_doi(deposition_id) - dataset.ds_meta_data.dataset_doi = deposition_doi - app.db.session.commit() + deposition_doi = zenodo_service.get_doi(deposition_id) + DSMetaDataRepository().update( + dataset.ds_meta_data_id, dataset_doi=deposition_doi + ) except Exception: pass @@ -81,173 +99,138 @@ def create_dataset(): # it has not been possible to create the deposition in Zenodo, so we save everything locally # create feature models - feature_models = create_feature_models_in_db(dataset, uploaded_models_data) + feature_models = create_feature_models_in_db( + dataset, uploaded_models_data + ) # move feature models permanently move_feature_models(dataset.id, feature_models) - pass - return jsonify({'message': zenodo_response_json}), 200 + return jsonify({"message": zenodo_response_json}), 200 except Exception as e: - return jsonify({'message': str(e)}), 500 + return jsonify({"message": str(e)}), 500 # Delete temp folder - file_path = os.path.join(app.upload_folder_name(), 'temp', str(current_user.id)) + file_path = os.path.join(app.upload_folder_name(), "temp", str(current_user.id)) if os.path.exists(file_path) and os.path.isdir(file_path): shutil.rmtree(file_path) - return render_template('dataset/upload_dataset.html', form=form) + return render_template("dataset/upload_dataset.html", form=form) -@dataset_bp.route('/dataset/list', methods=['GET', 'POST']) +@dataset_bp.route("/dataset/list", methods=["GET", "POST"]) @login_required def list_dataset(): - # synchronized datasets - datasets = DataSet.query.join(DSMetaData).filter( - DataSet.user_id == current_user.id, - DSMetaData.dataset_doi.isnot(None) - ).order_by(DataSet.created_at.desc()).all() - - # local datasets - local_datasets = DataSet.query.join(DSMetaData).filter( - DataSet.user_id == current_user.id, - DSMetaData.dataset_doi.is_(None) - ).order_by(DataSet.created_at.desc()).all() - - return render_template('dataset/list_datasets.html', datasets=datasets, local_datasets=local_datasets) + return render_template( + "dataset/list_datasets.html", + datasets=DataSetRepository().get_synchronized(current_user.id), + local_datasets=DataSetRepository().get_unsynchronized(current_user.id), + ) def create_dataset_in_db(basic_info_data): - # get dataset metadata - title = basic_info_data["title"][0] - description = basic_info_data["description"][0] - publication_type = basic_info_data["publication_type"][0] - publication_doi = basic_info_data["publication_doi"][0] - tags = basic_info_data["tags"][0] - - # create dataset metadata - ds_meta_data = DSMetaData( - title=title, - description=description, - publication_type=PublicationType(publication_type), - publication_doi=publication_doi, - tags=tags - ) - app.db.session.add(ds_meta_data) - app.db.session.commit() + ds_data = { + "title": basic_info_data["title"][0], + "description": basic_info_data["description"][0], + "publication_type": PublicationType(basic_info_data["publication_type"][0]), + "publication_doi": basic_info_data["publication_doi"][0], + "tags": basic_info_data["tags"][0], + } + ds_meta_data = DSMetaDataRepository().create(**ds_data) # create dataset metadata authors - # I always add myself - author = Author( - name=f"{current_user.profile.surname}, {current_user.profile.name}", - affiliation=current_user.profile.affiliation if hasattr(current_user.profile, 'affiliation') else None, - orcid=current_user.profile.orcid if hasattr(current_user.profile, 'orcid') else None, - ds_meta_data_id=ds_meta_data.id - ) - app.db.session.add(author) - app.db.session.commit() + author_data = { + "name": f"{current_user.profile.surname}, {current_user.profile.name}", + "affiliation": current_user.profile.affiliation + if hasattr(current_user.profile, "affiliation") + else None, + "orcid": current_user.profile.orcid + if hasattr(current_user.profile, "orcid") + else None, + "ds_meta_data_id": ds_meta_data.id, + } + AuthorRepository().create(**author_data) # how many authors are there? if "author_name" in basic_info_data: number_of_authors = len(basic_info_data["author_name"]) for i in range(number_of_authors): - author_name = basic_info_data["author_name"][i] - author_affiliation = basic_info_data["author_affiliation"][i] - author_orcid = basic_info_data["author_orcid"][i] - - author = Author( - name=author_name, - affiliation=author_affiliation, - orcid=author_orcid, - ds_meta_data_id=ds_meta_data.id - ) - app.db.session.add(author) - app.db.session.commit() + extra_author = { + "name": basic_info_data["author_name"][i], + "affiliation": basic_info_data["author_affiliation"][i], + "orcid": basic_info_data["author_orcid"][i], + "ds_meta_data_id": ds_meta_data.id, + } + AuthorRepository().create(**extra_author) # create dataset - dataset = DataSet(user_id=current_user.id, ds_meta_data_id=ds_meta_data.id) - app.db.session.add(dataset) - app.db.session.commit() - - return dataset + return DataSetRepository().create( + user_id=current_user.id, ds_meta_data_id=ds_meta_data.id + ) def create_feature_models_in_db(dataset: DataSet, uploaded_models_data: dict): feature_models = [] - if "uvl_identifier" in uploaded_models_data: - - number_of_models = len(uploaded_models_data["uvl_identifier"]) - - for i in range(number_of_models): - - # get feature model metadata - uvl_identifier = uploaded_models_data["uvl_identifier"][i] - uvl_filename = uploaded_models_data["uvl_filename"][i] - title = uploaded_models_data["title"][i] - description = uploaded_models_data["description"][i] - uvl_publication_type = uploaded_models_data["uvl_publication_type"][i] - publication_doi = uploaded_models_data["publication_doi"][i] - tags = uploaded_models_data["tags"][i] - uvl_version = uploaded_models_data["uvl_version"][i] - - # create feature model metadata - feature_model_metadata = FMMetaData( - uvl_filename=uvl_filename, - title=title, - description=description, - publication_type=PublicationType(uvl_publication_type), - publication_doi=publication_doi, - tags=tags, - uvl_version=uvl_version - ) - app.db.session.add(feature_model_metadata) - app.db.session.commit() + for i, uvl_identifier in enumerate(uploaded_models_data.get("uvl_identifier", [])): + uvl_filename = uploaded_models_data["uvl_filename"][i] - # create feature model - feature_model = FeatureModel( - data_set_id=dataset.id, - fm_meta_data_id=feature_model_metadata.id - ) - app.db.session.add(feature_model) - app.db.session.commit() - - # associated authors in feature model - if f"author_name_{uvl_identifier}" in uploaded_models_data.keys(): - number_of_authors_in_model = len(uploaded_models_data[f"author_name_{uvl_identifier}"]) - for a in range(number_of_authors_in_model): - author = Author( - name=uploaded_models_data[f"author_name_{uvl_identifier}"][a], - affiliation=uploaded_models_data[f"author_affiliation_{uvl_identifier}"][a], - orcid=uploaded_models_data[f"author_orcid_{uvl_identifier}"][a], - fm_meta_data_id=feature_model_metadata.id - ) - app.db.session.add(author) - app.db.session.commit() - - # associated files in feature model - user_id = current_user.id - file_path = os.path.join(app.upload_folder_name(), 'temp', str(user_id), uvl_filename) - checksum, size = calculate_checksum_and_size(file_path) - file = File( - name=uvl_filename, - checksum=checksum, - size=size, - feature_model_id=feature_model.id + # create feature model metadata + feature_model_metadata = FMMetaDataRepository().create( + uvl_filename=uvl_filename, + title=uploaded_models_data["title"][i], + description=uploaded_models_data["description"][i], + publication_type=PublicationType( + uploaded_models_data["uvl_publication_type"][i] + ), + publication_doi=uploaded_models_data["publication_doi"][i], + tags=uploaded_models_data["tags"][i], + uvl_version=uploaded_models_data["uvl_version"][i], + ) + + # create feature model + feature_model = FeatureModelRepository().create( + data_set_id=dataset.id, + fm_meta_data_id=feature_model_metadata.id, + ) + + # associated authors in feature model + for idx, author_name in enumerate( + uploaded_models_data.get(f"author_name_{uvl_identifier}", []) + ): + AuthorRepository().create( + name=author_name, + affiliation=uploaded_models_data[ + f"author_affiliation_{uvl_identifier}" + ][idx], + orcid=uploaded_models_data[f"author_orcid_{uvl_identifier}"][idx], + fm_meta_data_id=feature_model_metadata.id, ) - app.db.session.add(file) - app.db.session.commit() - feature_models.append(feature_model) + # associated files in feature model + user_id = current_user.id + file_path = os.path.join( + app.upload_folder_name(), "temp", str(user_id), uvl_filename + ) + checksum, size = calculate_checksum_and_size(file_path) + + FileRepository().create( + name=uvl_filename, + checksum=checksum, + size=size, + feature_model_id=feature_model.id, + ) + + feature_models.append(feature_model) return feature_models def calculate_checksum_and_size(file_path): file_size = os.path.getsize(file_path) - with open(file_path, 'rb') as file: + with open(file_path, "rb") as file: content = file.read() hash_md5 = hashlib.md5(content).hexdigest() return hash_md5, file_size @@ -255,8 +238,8 @@ def calculate_checksum_and_size(file_path): def move_feature_models(dataset_id, feature_models, user=None): user_id = current_user.id if user is None else user.id - source_dir = f'uploads/temp/{user_id}/' - dest_dir = f'uploads/user_{user_id}/dataset_{dataset_id}/' + source_dir = f"uploads/temp/{user_id}/" + dest_dir = f"uploads/user_{user_id}/dataset_{dataset_id}/" os.makedirs(dest_dir, exist_ok=True) @@ -265,141 +248,152 @@ def move_feature_models(dataset_id, feature_models, user=None): shutil.move(os.path.join(source_dir, uvl_filename), dest_dir) -@dataset_bp.route('/dataset/file/upload', methods=['POST']) +@dataset_bp.route("/dataset/file/upload", methods=["POST"]) @login_required def upload(): - file = request.files['file'] + file = request.files["file"] user_id = current_user.id - temp_folder = os.path.join(app.upload_folder_name(), 'temp', str(user_id)) - - if file and file.filename.endswith('.uvl'): - - # create temp folder - if not os.path.exists(temp_folder): - os.makedirs(temp_folder) - - file_path = os.path.join(temp_folder, file.filename) - - if os.path.exists(file_path): - # Generate unique filename (by recursion) - base_name, extension = os.path.splitext(file.filename) - i = 1 - while os.path.exists(os.path.join(temp_folder, f"{base_name} ({i}){extension}")): - i += 1 - new_filename = f"{base_name} ({i}){extension}" - file_path = os.path.join(temp_folder, new_filename) - else: - new_filename = file.filename - - try: - file.save(file_path) - if True: - return jsonify({ - 'message': 'UVL uploaded and validated successfully', - 'filename': new_filename - }), 200 - else: - return jsonify({'message': 'No valid model'}), 400 - except Exception as e: - return jsonify({'message': str(e)}), 500 - + temp_folder = os.path.join(app.upload_folder_name(), "temp", str(user_id)) + + if not file or not file.filename.endswith(".uvl"): + return jsonify({"message": "No valid file"}), 400 + + # create temp folder + if not os.path.exists(temp_folder): + os.makedirs(temp_folder) + + file_path = os.path.join(temp_folder, file.filename) + + if os.path.exists(file_path): + # Generate unique filename (by recursion) + base_name, extension = os.path.splitext(file.filename) + i = 1 + while os.path.exists( + os.path.join(temp_folder, f"{base_name} ({i}){extension}") + ): + i += 1 + new_filename = f"{base_name} ({i}){extension}" + file_path = os.path.join(temp_folder, new_filename) else: - return jsonify({'message': 'No valid file'}), 400 + new_filename = file.filename + + try: + file.save(file_path) + if True: + return ( + jsonify( + { + "message": "UVL uploaded and validated successfully", + "filename": new_filename, + } + ), + 200, + ) + else: + return jsonify({"message": "No valid model"}), 400 + except Exception as e: + return jsonify({"message": str(e)}), 500 -@dataset_bp.route('/dataset/file/delete', methods=['POST']) +@dataset_bp.route("/dataset/file/delete", methods=["POST"]) def delete(): data = request.get_json() - filename = data.get('file') + filename = data.get("file") user_id = current_user.id - temp_folder = os.path.join(app.upload_folder_name(), 'temp', str(user_id)) + temp_folder = os.path.join(app.upload_folder_name(), "temp", str(user_id)) filepath = os.path.join(temp_folder, filename) if os.path.exists(filepath): os.remove(filepath) - return jsonify({'message': 'File deleted successfully'}) - else: - return jsonify({'error': 'Error: File not found'}) + return jsonify({"message": "File deleted successfully"}) + return jsonify({"error": "Error: File not found"}) -@dataset_bp.route('/dataset/download/', methods=['GET']) + +@dataset_bp.route("/dataset/download/", methods=["GET"]) def download_dataset(dataset_id): - dataset = DataSet.query.get_or_404(dataset_id) + dataset = DataSetRepository().get_or_404(dataset_id) file_path = f"uploads/user_{dataset.user_id}/dataset_{dataset.id}/" temp_dir = tempfile.mkdtemp() - zip_path = os.path.join(temp_dir, f'dataset_{dataset_id}.zip') + zip_path = os.path.join(temp_dir, f"dataset_{dataset_id}.zip") - with ZipFile(zip_path, 'w') as zipf: + with ZipFile(zip_path, "w") as zipf: for subdir, dirs, files in os.walk(file_path): for file in files: full_path = os.path.join(subdir, file) relative_path = os.path.relpath(full_path, file_path) - zipf.write(full_path, arcname=os.path.join(os.path.basename(zip_path[:-4]), relative_path)) + zipf.write( + full_path, + arcname=os.path.join( + os.path.basename(zip_path[:-4]), relative_path + ), + ) - user_cookie = request.cookies.get('download_cookie') + user_cookie = request.cookies.get("download_cookie") if not user_cookie: - user_cookie = str(uuid.uuid4()) # Generate a new unique identifier if it does not exist + user_cookie = str( + uuid.uuid4() + ) # Generate a new unique identifier if it does not exist # Save the cookie to the user's browser - resp = make_response(send_from_directory( - temp_dir, - f'dataset_{dataset_id}.zip', - as_attachment=True, - mimetype='application/zip' - )) - resp.set_cookie('download_cookie', user_cookie) + resp = make_response( + send_from_directory( + temp_dir, + f"dataset_{dataset_id}.zip", + as_attachment=True, + mimetype="application/zip", + ) + ) + resp.set_cookie("download_cookie", user_cookie) else: resp = send_from_directory( temp_dir, - f'dataset_{dataset_id}.zip', + f"dataset_{dataset_id}.zip", as_attachment=True, - mimetype='application/zip' + mimetype="application/zip", ) # Record the download in your database - download_record = DSDownloadRecord( + DSDownloadRecordRepository().create( user_id=current_user.id if current_user.is_authenticated else None, dataset_id=dataset_id, download_date=datetime.utcnow(), - download_cookie=user_cookie) - - app.db.session.add(download_record) - app.db.session.commit() + download_cookie=user_cookie, + ) return resp -@dataset_bp.route('/dataset/view/', methods=['GET']) +@dataset_bp.route("/dataset/view/", methods=["GET"]) def view_dataset(dataset_id): - dataset = DataSet.query.get_or_404(dataset_id) + dataset = DataSetRepository().get_or_404(dataset_id) # Get the cookie from the request or generate a new one if it does not exist - user_cookie = request.cookies.get('view_cookie') + user_cookie = request.cookies.get("view_cookie") if not user_cookie: user_cookie = str(uuid.uuid4()) # Record the view in your database - view_record = DSViewRecord( + DSViewRecordRepository().create( user_id=current_user.id if current_user.is_authenticated else None, dataset_id=dataset_id, view_date=datetime.utcnow(), - view_cookie=user_cookie) - app.db.session.add(view_record) - app.db.session.commit() + view_cookie=user_cookie, + ) # Save the cookie to the user's browser - resp = make_response(render_template('dataset/view_dataset.html', dataset=dataset)) - resp.set_cookie('view_cookie', user_cookie) + resp = make_response(render_template("dataset/view_dataset.html", dataset=dataset)) + resp.set_cookie("view_cookie", user_cookie) return resp -@dataset_bp.route('/file/download/', methods=['GET']) +@dataset_bp.route("/file/download/", methods=["GET"]) def download_file(file_id): - file = File.query.get_or_404(file_id) + file = FileRepository().get_or_404(file_id) filename = file.name directory_path = f"uploads/user_{file.feature_model.data_set.user_id}/dataset_{file.feature_model.data_set_id}/" @@ -407,306 +401,57 @@ def download_file(file_id): file_path = os.path.join(parent_directory_path, directory_path) # Get the cookie from the request or generate a new one if it does not exist - user_cookie = request.cookies.get('file_download_cookie') + user_cookie = request.cookies.get("file_download_cookie") if not user_cookie: user_cookie = str(uuid.uuid4()) # Record the download in your database - download_record = FileDownloadRecord( + FileDownloadRecordRepository().create( user_id=current_user.id if current_user.is_authenticated else None, file_id=file_id, download_date=datetime.utcnow(), - download_cookie=user_cookie) - app.db.session.add(download_record) - app.db.session.commit() + download_cookie=user_cookie, + ) # Save the cookie to the user's browser - resp = make_response(send_from_directory(directory=file_path, path=filename, as_attachment=True)) - resp.set_cookie('file_download_cookie', user_cookie) - - return resp - - -''' - API ENDPOINTS FOR DATASET MODEL -''' - -''' -@dataset_bp.route('/api/v1/dataset/', methods=['GET']) -def get_all_dataset(): - datasets = DataSet.query.order_by(DataSet.created_at.desc()).all() - dataset_list = [dataset.to_dict() for dataset in datasets] - return jsonify(dataset_list) - - -@dataset_bp.route('/api/v1/dataset/', methods=['GET']) -def get_dataset(dataset_id): - dataset = DataSet.query.get_or_404(dataset_id) - return dataset.to_dict() - - -@dataset_bp.route('/api/v1/dataset/', methods=['POST']) -def api_create_dataset(): - """ - ENDPOINT FOR CREATE DATASET - """ - - """ - PART 1: GET BASIC DATA - """ - - user = app.get_user_by_token("BLABLABLA") # TODO - data = json.loads(request.files['json'].read()) - temp_folder = os.path.join(app.upload_folder_name(), 'temp', str(user.id)) - - # Delete the existing temp_folder if it exists - if os.path.exists(temp_folder): - shutil.rmtree(temp_folder) - - # Create the new temp_folder - os.makedirs(temp_folder) - - info = data['info'] - models = data['models'] - - """ - PART 2: SAVE BASIC DATA - """ - ds_meta_data = _create_ds_meta_data(info=info) - _create_authors(info=info, ds_meta_data=ds_meta_data) - dataset = _create_dataset(user=user, ds_meta_data=ds_meta_data) - - """ - PART 3: SAVE FILES IN TEMP FOLDER - """ - files = request.files.to_dict() - for filename, file in files.items(): - if filename != 'json': - - if file and filename.endswith('.uvl'): - - # create temporal folder for this user - if not os.path.exists(temp_folder): - os.makedirs(temp_folder) - - try: - filename = os.path.basename(file.filename) - file.save(os.path.join(temp_folder, filename)) - # TODO: Change valid model function - if True: - continue # TODO - else: - dataset.delete() - return jsonify({'message': f'{filename} is not a valid model'}), 400 - except Exception as e: - dataset.delete() - return jsonify({'Exception in save files in temp folder': str(e)}), 500 - - else: - dataset.delete() - return jsonify({'message': f'{filename} is not a valid extension'}), 400 - - """ - PART 4: SEND BASIC DATA TO ZENODO - """ - zenodo_response_json = zenodo_create_new_deposition(dataset) - response_data = json.dumps(zenodo_response_json) - zenodo_json_data = json.loads(response_data) - - """ - PART 5: CREATE FEATURE MODELS - """ - feature_models = _create_feature_models(dataset=dataset, models=models, user=user) - - if zenodo_json_data.get('conceptrecid'): - - # update dataset with deposition id in Zenodo - deposition_id = zenodo_json_data.get('id') - dataset.ds_meta_data.deposition_id = deposition_id - app.db.session.commit() - - """ - PART 6: SEND FILES TO ZENODO AND PUBLISH - """ - try: - # iterate for each feature model (one feature model = one request to Zenodo - try: - for feature_model in feature_models: - zenodo_upload_file(deposition_id, feature_model, user=user) - - # Wait for 0.6 seconds before the next API call to ensure we do not exceed - # the rate limit of 100 requests per minute. This is because 60 seconds (1 minute) - # divided by 100 requests equals 0.6 seconds per request. - time.sleep(0.6) - - except Exception as e: - logging.error("Exception occurred during file upload", exc_info=True) - return jsonify({'exception': str(e)}), 500 - - # publish deposition - try: - zenodo_publish_deposition(deposition_id) - except Exception as e: - logging.error("Exception occurred during publish deposition", exc_info=True) - return jsonify({'exception': str(e)}), 500 - - # update DOI - try: - deposition_doi = zenodo_get_doi(deposition_id) - dataset.ds_meta_data.dataset_doi = deposition_doi - app.db.session.commit() - except Exception as e: - logging.error("Exception occurred during update DOI", exc_info=True) - return jsonify({'exception': str(e)}), 500 - - except Exception as e: - logging.error("Exception occurred", exc_info=True) - return jsonify({'exception': str(e)}), 500 - - """ - PART 7: MOVE FEATURE MODELS PERMANENTLY - """ - move_feature_models(dataset.id, feature_models, user=user) - - return jsonify(dataset.to_dict()), 200 - - -def _create_ds_meta_data(info: dict) -> DSMetaData: - ds_meta_data = DSMetaData( - title=info["title"], - description=info["description"], - publication_type=PublicationType(info["publication_type"]), - publication_doi=info["publication_doi"], - tags=','.join(tag.strip() for tag in info['tags']) + resp = make_response( + send_from_directory(directory=file_path, path=filename, as_attachment=True) ) - app.db.session.add(ds_meta_data) - app.db.session.commit() - - return ds_meta_data - - -def _create_authors(info: dict, ds_meta_data: DSMetaData) -> List[Author]: - authors = [] - - authors_info = info.get("authors") - if authors_info: - for author_info in authors_info: - author = Author( - name=author_info.get("name"), - affiliation=author_info.get("affiliation"), - orcid=author_info.get("orcid", None), - ds_meta_data_id=ds_meta_data.id - ) - authors.append(author) - app.db.session.add(author) - - app.db.session.commit() - - return authors - + resp.set_cookie("file_download_cookie", user_cookie) -def _create_dataset(user: User, ds_meta_data: DSMetaData) -> DataSet: - dataset = DataSet(user_id=user.id, ds_meta_data_id=ds_meta_data.id) - app.db.session.add(dataset) - app.db.session.commit() - - return dataset - - -def _create_feature_models(dataset: DataSet, models: dict, user: User) -> List[FeatureModel]: - feature_models = [] - - for model in models: - - filename = os.path.basename(model['filename']) # only name of file with .uvl extension - title = model.get('title', '') - description = model.get('description', '') - publication_type = model.get('publication_type', 'none') - publication_doi = model.get('publication_doi', '') - tags = ','.join(tag.strip() for tag in model.get('tags', [])) - - # create feature model metadata - feature_model_metadata = FMMetaData( - uvl_filename=filename, - title=title, - description=description, - publication_type=publication_type, - publication_doi=publication_doi, - tags=tags - ) - app.db.session.add(feature_model_metadata) - app.db.session.commit() - - # associated authors in feature model - if 'authors' in model and isinstance(model['authors'], list): - for author_data in model['authors']: - if 'name' in author_data: - author = Author( - name=author_data.get('name'), - affiliation=author_data.get('affiliation'), - orcid=author_data.get('orcid'), - fm_meta_data_id=feature_model_metadata.id - ) - app.db.session.add(author) - app.db.session.commit() - - # create feature model - feature_model = FeatureModel( - data_set_id=dataset.id, - fm_meta_data_id=feature_model_metadata.id - ) - app.db.session.add(feature_model) - app.db.session.commit() - - # associated files in feature model - user_id = user.id - file_path = os.path.join(app.upload_folder_name(), 'temp', str(user_id), filename) - checksum, size = calculate_checksum_and_size(file_path) - file = File( - name=filename, - checksum=checksum, - size=size, - feature_model_id=feature_model.id - ) - app.db.session.add(file) - app.db.session.commit() - - feature_models.append(feature_model) - - return feature_models - -''' + return resp -@dataset_bp.route('/doi//', methods=['GET']) +@dataset_bp.route("/doi//", methods=["GET"]) def subdomain_index(doi): # Busca el dataset por DOI - ds_meta_data = DSMetaData.query.filter_by(dataset_doi=doi).first() - if ds_meta_data: - dataset = ds_meta_data.data_set - - if dataset: - dataset_id = dataset.id - user_cookie = request.cookies.get('view_cookie', str(uuid.uuid4())) - - # Registra la vista del dataset - view_record = DSViewRecord( - user_id=current_user.id if current_user.is_authenticated else None, - dataset_id=dataset_id, - view_date=datetime.utcnow(), - view_cookie=user_cookie - ) - app.db.session.add(view_record) - app.db.session.commit() - - # Prepara la respuesta y establece la cookie - resp = make_response(render_template('dataset/view_dataset.html', dataset=dataset)) - resp.set_cookie('view_cookie', user_cookie, max_age=30 * 24 * 60 * 60) # Ejemplo: cookie expira en 30 días + ds_meta_data = DSMetaDataRepository().filter_by_doi(doi) + if not ds_meta_data: + abort(404) + + dataset = ds_meta_data.data_set + if dataset: + dataset_id = dataset.id + user_cookie = request.cookies.get("view_cookie", str(uuid.uuid4())) + + # Registra la vista del dataset + DSViewRecordRepository().create( + user_id=current_user.id if current_user.is_authenticated else None, + dataset_id=dataset_id, + view_date=datetime.utcnow(), + view_cookie=user_cookie, + ) - return resp - else: - # Aquí puedes manejar el caso de que el DOI no corresponda a un dataset existente - # Por ejemplo, mostrar un error 404 o redirigir a una página de error - return "Dataset no encontrado", 404 + # Prepara la respuesta y establece la cookie + resp = make_response( + render_template("dataset/view_dataset.html", dataset=dataset) + ) + resp.set_cookie( + "view_cookie", user_cookie, max_age=30 * 24 * 60 * 60 + ) # Ejemplo: cookie expira en 30 días - abort(404) + return resp + else: + # Aquí puedes manejar el caso de que el DOI no corresponda a un dataset existente + # Por ejemplo, mostrar un error 404 o redirigir a una página de error + return "Dataset no encontrado", 404 diff --git a/app/blueprints/zenodo/routes.py b/app/blueprints/zenodo/routes.py index e3a4ba8e8..1506d5fb7 100644 --- a/app/blueprints/zenodo/routes.py +++ b/app/blueprints/zenodo/routes.py @@ -1,7 +1,15 @@ from flask import render_template + from app.blueprints.zenodo import zenodo_bp +from app.blueprints.zenodo.services import ZenodoService @zenodo_bp.route('/zenodo', methods=['GET']) def index(): return render_template('zenodo/index.html') + + +@zenodo_bp.route('/zenodo/test', methods=['GET']) +def zenodo_test() -> dict: + service = ZenodoService() + return service.test_full_connection() diff --git a/app/blueprints/zenodo/services.py b/app/blueprints/zenodo/services.py index 3e3febbc3..4b8308725 100644 --- a/app/blueprints/zenodo/services.py +++ b/app/blueprints/zenodo/services.py @@ -1,8 +1,4 @@ -from app.blueprints.zenodo.repositories import ZenodoRepository -from core.services.BaseService import BaseService - import os - import requests from dotenv import load_dotenv @@ -11,215 +7,239 @@ import app from app.blueprints.dataset.models import DataSet, FeatureModel +from app.blueprints.zenodo.repositories import ZenodoRepository +from core.services.BaseService import BaseService -load_dotenv() - -ZENODO_API_URL = 'https://sandbox.zenodo.org/api/deposit/depositions' -ZENODO_ACCESS_TOKEN = os.getenv('ZENODO_ACCESS_TOKEN') - - -def test_zenodo_connection() -> bool: - """ - Test the connection with Zenodo. - - Returns: - bool: True if the connection is successful, False otherwise. - """ - headers = {"Content-Type": "application/json"} - params = {'access_token': ZENODO_ACCESS_TOKEN} - response = requests.get(ZENODO_API_URL, params=params, headers=headers) - return response.status_code == 200 - - -def test_full_zenodo_connection() -> Response: - """ - Test the connection with Zenodo by creating a deposition, uploading an empty test file, and deleting the deposition. - - Returns: - bool: True if the connection, upload, and deletion are successful, False otherwise. - """ - success = True +load_dotenv() - # Create an empty file - file_path = os.path.join(current_app.root_path, "test_file.txt") - with open(file_path, 'w'): - pass - messages = [] # List to store messages +class ZenodoService(BaseService): + ZENODO_API_URL = os.getenv( + "ZENODO_API_URL", "https://sandbox.zenodo.org/api/deposit/depositions" + ) + ZENODO_ACCESS_TOKEN = os.getenv("ZENODO_ACCESS_TOKEN") - # Step 1: Create a deposition on Zenodo - headers = {"Content-Type": "application/json"} - data = { - "metadata": { - "title": "Test Deposition", - "upload_type": "dataset", - "description": "This is a test deposition created via Zenodo API", - "creators": [{"name": "John Doe"}] - } - } - - params = {'access_token': ZENODO_ACCESS_TOKEN} - response = requests.post(ZENODO_API_URL, json=data, params=params, headers=headers) - - if response.status_code != 201: - messages.append(f"Failed to create test deposition on Zenodo. Response code: {response.status_code}") - success = False - - deposition_id = response.json()["id"] - - # Step 2: Upload an empty file to the deposition - data = {'name': "test_file.txt"} - file_path = os.path.join(current_app.root_path, "test_file.txt") - files = {'file': open(file_path, 'rb')} - publish_url = f'{ZENODO_API_URL}/{deposition_id}/files' - response = requests.post(publish_url, params=params, data=data, files=files) - - if response.status_code != 201: - messages.append(f"Failed to upload test file to Zenodo. Response code: {response.status_code}") - success = False - - # Step 3: Delete the deposition - response = requests.delete(f"{ZENODO_API_URL}/{deposition_id}", params=params) - - if os.path.exists(file_path): - os.remove(file_path) - - return jsonify({"success": success, "messages": messages}) - - -def get_all_depositions() -> dict: - """ - Get all depositions from Zenodo. - - Returns: - dict: The response in JSON format with the depositions. - """ - headers = {"Content-Type": "application/json"} - params = {'access_token': ZENODO_ACCESS_TOKEN} - response = requests.get(ZENODO_API_URL, params=params, headers=headers) - if response.status_code != 200: - raise Exception('Failed to get depositions') - return response.json() - - -def zenodo_create_new_deposition(dataset: DataSet) -> dict: - """ - Create a new deposition in Zenodo. - - Args: - dataset (DataSet): The DataSet object containing the metadata of the deposition. - - Returns: - dict: The response in JSON format with the details of the created deposition. - """ - metadata = { - 'title': dataset.ds_meta_data.title, - 'upload_type': 'dataset' if dataset.ds_meta_data.publication_type.value == "none" else 'publication', - 'publication_type': dataset.ds_meta_data.publication_type.value - if dataset.ds_meta_data.publication_type.value != "none" else None, - 'description': dataset.ds_meta_data.description, - 'creators': [{ - 'name': author.name, - **({'affiliation': author.affiliation} if author.affiliation else {}), - **({'orcid': author.orcid} if author.orcid else {}) - } for author in dataset.ds_meta_data.authors], - 'keywords': ["uvlhub"] if not dataset.ds_meta_data.tags else dataset.ds_meta_data.tags.split(", ") + ["uvlhub"], - 'access_right': 'open', - 'license': 'CC-BY-4.0' - } - - data = { - 'metadata': metadata - } - - headers = {"Content-Type": "application/json"} - params = {'access_token': ZENODO_ACCESS_TOKEN} - response = requests.post(ZENODO_API_URL, params=params, json=data, headers=headers) - if response.status_code != 201: - error_message = f'Failed to create deposition. Error details: {response.json()}' - raise Exception(error_message) - return response.json() - - -def zenodo_upload_file(deposition_id: int, feature_model: FeatureModel, user=None) -> dict: - """ - Upload a file to a deposition in Zenodo. - - Args: - deposition_id (int): The ID of the deposition in Zenodo. - feature_model (FeatureModel): The FeatureModel object representing the feature model. - user (FeatureModel): The User object representing the file owner. - - Returns: - dict: The response in JSON format with the details of the uploaded file. - """ - uvl_filename = feature_model.fm_meta_data.uvl_filename - data = {'name': uvl_filename} - user_id = current_user.id if user is None else user.id - file_path = os.path.join(app.upload_folder_name(), 'temp', str(user_id), uvl_filename) - files = {'file': open(file_path, 'rb')} - - publish_url = f'{ZENODO_API_URL}/{deposition_id}/files' - params = {'access_token': ZENODO_ACCESS_TOKEN} - response = requests.post(publish_url, params=params, data=data, files=files) - if response.status_code != 201: - error_message = f'Failed to upload files. Error details: {response.json()}' - raise Exception(error_message) - return response.json() - - -def zenodo_publish_deposition(deposition_id: int) -> dict: - """ - Publish a deposition in Zenodo. - - Args: - deposition_id (int): The ID of the deposition in Zenodo. - - Returns: - dict: The response in JSON format with the details of the published deposition. - """ - headers = {"Content-Type": "application/json"} - publish_url = f'{ZENODO_API_URL}/{deposition_id}/actions/publish' - params = {'access_token': ZENODO_ACCESS_TOKEN} - response = requests.post(publish_url, params=params, headers=headers) - if response.status_code != 202: - raise Exception('Failed to publish deposition') - return response.json() - - -def zenodo_get_deposition(deposition_id: int) -> dict: - """ - Get a deposition from Zenodo. - - Args: - deposition_id (int): The ID of the deposition in Zenodo. - - Returns: - dict: The response in JSON format with the details of the deposition. - """ - headers = {"Content-Type": "application/json"} - deposition_url = f'{ZENODO_API_URL}/{deposition_id}' - params = {'access_token': ZENODO_ACCESS_TOKEN} - response = requests.get(deposition_url, params=params, headers=headers) - if response.status_code != 200: - raise Exception('Failed to get deposition') - return response.json() - - -def zenodo_get_doi(deposition_id: int) -> str: - """ - Get the DOI of a deposition from Zenodo. - - Args: - deposition_id (int): The ID of the deposition in Zenodo. - - Returns: - str: The DOI of the deposition. - """ - return zenodo_get_deposition(deposition_id).get('doi') - - -class Zenodo(BaseService): def __init__(self): super().__init__(ZenodoRepository()) + self.headers = {"Content-Type": "application/json"} + self.params = {"access_token": self.ZENODO_ACCESS_TOKEN} + + def test_connection(self) -> bool: + """ + Test the connection with Zenodo. + + Returns: + bool: True if the connection is successful, False otherwise. + """ + response = requests.get( + self.ZENODO_API_URL, params=self.params, headers=self.headers + ) + return response.status_code == 200 + + def test_full_connection(self) -> Response: + """ + Test the connection with Zenodo by creating a deposition, uploading an empty test file, and deleting the deposition. + + Returns: + bool: True if the connection, upload, and deletion are successful, False otherwise. + """ + + success = True + + # Create an empty file + file_path = os.path.join(current_app.root_path, "test_file.txt") + with open(file_path, "w"): + pass + + messages = [] # List to store messages + + # Step 1: Create a deposition on Zenodo + data = { + "metadata": { + "title": "Test Deposition", + "upload_type": "dataset", + "description": "This is a test deposition created via Zenodo API", + "creators": [{"name": "John Doe"}], + } + } + + response = requests.post( + self.ZENODO_API_URL, json=data, params=self.params, headers=self.headers + ) + + if response.status_code != 201: + return jsonify( + { + "success": False, + "messages": f"Failed to create test deposition on Zenodo. Response code: {response.status_code}", + } + ) + + deposition_id = response.json()["id"] + + # Step 2: Upload an empty file to the deposition + data = {"name": "test_file.txt"} + file_path = os.path.join(current_app.root_path, "test_file.txt") + files = {"file": open(file_path, "rb")} + publish_url = f"{self.ZENODO_API_URL}/{deposition_id}/files" + response = requests.post( + publish_url, params=self.params, data=data, files=files + ) + + if response.status_code != 201: + messages.append( + f"Failed to upload test file to Zenodo. Response code: {response.status_code}" + ) + success = False + + # Step 3: Delete the deposition + response = requests.delete( + f"{self.ZENODO_API_URL}/{deposition_id}", params=self.params + ) + + if os.path.exists(file_path): + os.remove(file_path) + + return jsonify({"success": success, "messages": messages}) + + def get_all_depositions(self) -> dict: + """ + Get all depositions from Zenodo. + + Returns: + dict: The response in JSON format with the depositions. + """ + response = requests.get( + self.ZENODO_API_URL, params=self.params, headers=self.headers + ) + if response.status_code != 200: + raise Exception("Failed to get depositions") + return response.json() + + def create_new_deposition(self, dataset: DataSet) -> dict: + """ + Create a new deposition in Zenodo. + + Args: + dataset (DataSet): The DataSet object containing the metadata of the deposition. + + Returns: + dict: The response in JSON format with the details of the created deposition. + """ + metadata = { + "title": dataset.ds_meta_data.title, + "upload_type": "dataset" + if dataset.ds_meta_data.publication_type.value == "none" + else "publication", + "publication_type": dataset.ds_meta_data.publication_type.value + if dataset.ds_meta_data.publication_type.value != "none" + else None, + "description": dataset.ds_meta_data.description, + "creators": [ + { + "name": author.name, + **( + {"affiliation": author.affiliation} + if author.affiliation + else {} + ), + **({"orcid": author.orcid} if author.orcid else {}), + } + for author in dataset.ds_meta_data.authors + ], + "keywords": ["uvlhub"] + if not dataset.ds_meta_data.tags + else dataset.ds_meta_data.tags.split(", ") + ["uvlhub"], + "access_right": "open", + "license": "CC-BY-4.0", + } + + data = {"metadata": metadata} + + response = requests.post( + self.ZENODO_API_URL, params=self.params, json=data, headers=self.headers + ) + if response.status_code != 201: + error_message = ( + f"Failed to create deposition. Error details: {response.json()}" + ) + raise Exception(error_message) + return response.json() + + def upload_file( + self, deposition_id: int, feature_model: FeatureModel, user=None + ) -> dict: + """ + Upload a file to a deposition in Zenodo. + + Args: + deposition_id (int): The ID of the deposition in Zenodo. + feature_model (FeatureModel): The FeatureModel object representing the feature model. + user (FeatureModel): The User object representing the file owner. + + Returns: + dict: The response in JSON format with the details of the uploaded file. + """ + uvl_filename = feature_model.fm_meta_data.uvl_filename + data = {"name": uvl_filename} + user_id = current_user.id if user is None else user.id + file_path = os.path.join( + app.upload_folder_name(), "temp", str(user_id), uvl_filename + ) + files = {"file": open(file_path, "rb")} + + publish_url = f"{self.ZENODO_API_URL}/{deposition_id}/files" + response = requests.post( + publish_url, params=self.params, data=data, files=files + ) + if response.status_code != 201: + error_message = f"Failed to upload files. Error details: {response.json()}" + raise Exception(error_message) + return response.json() + + def publish_deposition(self, deposition_id: int) -> dict: + """ + Publish a deposition in Zenodo. + + Args: + deposition_id (int): The ID of the deposition in Zenodo. + + Returns: + dict: The response in JSON format with the details of the published deposition. + """ + publish_url = f"{self.ZENODO_API_URL}/{deposition_id}/actions/publish" + response = requests.post(publish_url, params=self.params, headers=self.headers) + if response.status_code != 202: + raise Exception("Failed to publish deposition") + return response.json() + + def get_deposition(self, deposition_id: int) -> dict: + """ + Get a deposition from Zenodo. + + Args: + deposition_id (int): The ID of the deposition in Zenodo. + + Returns: + dict: The response in JSON format with the details of the deposition. + """ + deposition_url = f"{self.ZENODO_API_URL}/{deposition_id}" + response = requests.get( + deposition_url, params=self.params, headers=self.headers + ) + if response.status_code != 200: + raise Exception("Failed to get deposition") + return response.json() + + def get_doi(self, deposition_id: int) -> str: + """ + Get the DOI of a deposition from Zenodo. + + Args: + deposition_id (int): The ID of the deposition in Zenodo. + + Returns: + str: The DOI of the deposition. + """ + return self.get_deposition(deposition_id).get("doi") diff --git a/core/repositories/BaseRepository.py b/core/repositories/BaseRepository.py index d46428380..95029c48a 100644 --- a/core/repositories/BaseRepository.py +++ b/core/repositories/BaseRepository.py @@ -1,4 +1,4 @@ -from typing import TypeVar, Generic, Optional +from typing import Generic, NoReturn, Optional, TypeVar, Union import app @@ -19,6 +19,9 @@ def get_by_id(self, id: int) -> Optional[T]: instance: Optional[T] = self.model.query.get(id) return instance + def get_or_404(self, id: int) -> Union[T, NoReturn]: + return self.model.query.get_or_404(id) + def update(self, id: int, **kwargs) -> Optional[T]: instance: Optional[T] = self.get_by_id(id) if instance: From b530c93cd91d4b250515cfce6c74c153ed385ee3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?V=C3=ADctor=20Ram=C3=ADrez=20de=20la=20Corte?= Date: Thu, 6 Jun 2024 13:27:40 +0200 Subject: [PATCH 2/7] refactor: create services and repositories and replace in routes --- .flake8 | 2 +- app/__init__.py | 12 ---- app/blueprints/auth/models.py | 12 ---- app/blueprints/auth/repositories.py | 19 ++++++ app/blueprints/auth/routes.py | 58 +++++++---------- app/blueprints/auth/services.py | 15 +++-- app/blueprints/auth/tests/test_unit.py | 61 +++++++++++------ app/blueprints/dataset/repositories.py | 71 +++++++++++++++++--- app/blueprints/dataset/routes.py | 1 + app/blueprints/dataset/services.py | 7 ++ app/blueprints/explore/routes.py | 69 ++------------------ app/blueprints/profile/routes.py | 23 ++++--- app/blueprints/profile/services.py | 4 +- app/blueprints/profile/tests/test_unit.py | 19 ++++-- app/blueprints/public/routes.py | 28 ++++---- app/blueprints/pytest.ini | 2 +- app/blueprints/zenodo/services.py | 79 +++++++---------------- core/repositories/BaseRepository.py | 3 + 18 files changed, 239 insertions(+), 246 deletions(-) create mode 100644 app/blueprints/auth/repositories.py create mode 100644 app/blueprints/dataset/services.py diff --git a/.flake8 b/.flake8 index 79a16af7e..6deafc261 100644 --- a/.flake8 +++ b/.flake8 @@ -1,2 +1,2 @@ [flake8] -max-line-length = 120 \ No newline at end of file +max-line-length = 120 diff --git a/app/__init__.py b/app/__init__.py index 36de9ab2a..f13bfbcfa 100644 --- a/app/__init__.py +++ b/app/__init__.py @@ -94,16 +94,4 @@ def get_authenticated_user(): return None -def datasets_counter() -> int: - from app.blueprints.dataset.models import DataSet - count = DataSet.query.count() - return count - - -def feature_models_counter() -> int: - from app.blueprints.dataset.models import FeatureModel - count = FeatureModel.query.count() - return count - - app = create_app() diff --git a/app/blueprints/auth/models.py b/app/blueprints/auth/models.py index b70f14b3d..a14eda02b 100644 --- a/app/blueprints/auth/models.py +++ b/app/blueprints/auth/models.py @@ -38,15 +38,3 @@ def save(self): def delete(self): db.session.delete(self) db.session.commit() - - @staticmethod - def get_by_id(user_id): - return User.query.get(user_id) - - @staticmethod - def get_by_email(email): - return User.query.filter_by(email=email).first() - - @staticmethod - def get_all(): - return User.query.all() diff --git a/app/blueprints/auth/repositories.py b/app/blueprints/auth/repositories.py new file mode 100644 index 000000000..fadd6f76c --- /dev/null +++ b/app/blueprints/auth/repositories.py @@ -0,0 +1,19 @@ +import app +from app.blueprints.auth.models import User +from core.repositories.BaseRepository import BaseRepository + + +class UserRepository(BaseRepository): + def __init__(self): + super().__init__(User) + + def create(self, **kwargs): + password = kwargs.pop("password") + instance = self.model(**kwargs) + instance.set_password(password) + app.db.session.add(instance) + app.db.session.commit() + return instance + + def get_by_email(self, email: str): + return self.model.query.filter_by(email=email).first() diff --git a/app/blueprints/auth/routes.py b/app/blueprints/auth/routes.py index b8bfae92f..5d4d9a903 100644 --- a/app/blueprints/auth/routes.py +++ b/app/blueprints/auth/routes.py @@ -1,60 +1,48 @@ -from flask import (render_template, redirect, url_for, request) +from flask import render_template, redirect, url_for, request from flask_login import current_user, login_user, logout_user from app.blueprints.auth import auth_bp from app.blueprints.auth.forms import SignupForm, LoginForm from app.blueprints.auth.services import AuthenticationService +from app.blueprints.profile.services import UserProfileService -from app.blueprints.profile.models import UserProfile + +authentication_service = AuthenticationService() +user_profile_service = UserProfileService() @auth_bp.route("/signup/", methods=["GET", "POST"]) def show_signup_form(): if current_user.is_authenticated: return redirect(url_for('public.index')) + form = SignupForm() - error = None if form.validate_on_submit(): - name = form.name.data - surname = form.surname.data email = form.email.data - password = form.password.data - - from app.blueprints.auth.models import User - user = User.get_by_email(email) - if user is not None: - error = f'Email {email} in use' - else: - # Create user - user = User(email=email) - user.set_password(password) - user.save() - - # Create user profile - profile = UserProfile(name=name, surname=surname) - profile.user_id = user.id - profile.save() - - # Log user - login_user(user, remember=True) - return redirect(url_for('public.index')) - return render_template("auth/signup_form.html", form=form, error=error) + if not authentication_service.is_email_available(email): + return render_template("auth/signup_form.html", form=form, error=f'Email {email} in use') + + user = authentication_service.create(email=email, password=form.password.data) + user_profile_service.create(name=form.name.data, surname=form.surname.data, user_id=user.id) + + # Log user + login_user(user, remember=True) + return redirect(url_for('public.index')) + + return render_template("auth/signup_form.html", form=form) @auth_bp.route('/login', methods=['GET', 'POST']) def login(): if current_user.is_authenticated: return redirect(url_for('public.index')) + form = LoginForm() - if request.method == 'POST': - if form.validate_on_submit(): - email = form.email.data - password = form.password.data - if AuthenticationService.login(email, password): - return redirect(url_for('public.index')) - else: - error = 'Invalid credentials' - return render_template("auth/login_form.html", form=form, error=error) + if request.method == 'POST' and form.validate_on_submit(): + if authentication_service.login(form.email.data, form.password.data): + return redirect(url_for('public.index')) + + return render_template("auth/login_form.html", form=form, error='Invalid credentials') return render_template('auth/login_form.html', form=form) diff --git a/app/blueprints/auth/services.py b/app/blueprints/auth/services.py index 0032bbc6b..1533ffc27 100644 --- a/app/blueprints/auth/services.py +++ b/app/blueprints/auth/services.py @@ -1,14 +1,19 @@ from flask_login import login_user -from app.blueprints.auth.models import User +from app.blueprints.auth.repositories import UserRepository +from core.services.BaseService import BaseService -class AuthenticationService: +class AuthenticationService(BaseService): + def __init__(self): + super().__init__(UserRepository()) - @staticmethod - def login(email, password, remember=True): - user = User.get_by_email(email) + def login(self, email, password, remember=True): + user = self.repository.get_by_email(email) if user is not None and user.check_password(password): login_user(user, remember=remember) return True return False + + def is_email_available(self, email: str) -> bool: + return self.repository.get_by_email(email) is None diff --git a/app/blueprints/auth/tests/test_unit.py b/app/blueprints/auth/tests/test_unit.py index c4b08e36d..65c3aa8d1 100644 --- a/app/blueprints/auth/tests/test_unit.py +++ b/app/blueprints/auth/tests/test_unit.py @@ -2,7 +2,7 @@ from flask import url_for -@pytest.fixture(scope='module') +@pytest.fixture(scope="module") def test_client(test_client): """ Extends the test_client fixture to add additional specific data for module testing. @@ -17,33 +17,56 @@ def test_client(test_client): def test_login_success(test_client): - response = test_client.post('/login', data=dict( - email='test@example.com', - password='test1234' - ), follow_redirects=True) + response = test_client.post( + "/login", data=dict(email="test@example.com", password="test1234"), follow_redirects=True + ) - assert response.request.path != url_for('auth.login'), "Login was unsuccessful" + assert response.request.path != url_for("auth.login"), "Login was unsuccessful" - test_client.get('/logout', follow_redirects=True) + test_client.get("/logout", follow_redirects=True) def test_login_unsuccessful_bad_email(test_client): - response = test_client.post('/login', data=dict( - email='bademail@example.com', - password='test1234' - ), follow_redirects=True) + response = test_client.post( + "/login", data=dict(email="bademail@example.com", password="test1234"), follow_redirects=True + ) - assert response.request.path == url_for('auth.login'), "Login was unsuccessful" + assert response.request.path == url_for("auth.login"), "Login was unsuccessful" - test_client.get('/logout', follow_redirects=True) + test_client.get("/logout", follow_redirects=True) def test_login_unsuccessful_bad_password(test_client): - response = test_client.post('/login', data=dict( - email='test@example.com', - password='basspassword' - ), follow_redirects=True) + response = test_client.post( + "/login", data=dict(email="test@example.com", password="basspassword"), follow_redirects=True + ) - assert response.request.path == url_for('auth.login'), "Login was unsuccessful" + assert response.request.path == url_for("auth.login"), "Login was unsuccessful" - test_client.get('/logout', follow_redirects=True) + test_client.get("/logout", follow_redirects=True) + + +def test_signup_user_no_name(test_client): + response = test_client.post( + "/signup", data=dict(surname="Foo", email="test@example.com", password="test1234"), follow_redirects=True + ) + assert response.request.path == url_for("auth.show_signup_form"), "Signup was unsuccessful" + assert b"This field is required" in response.data, response.data + + +def test_signup_user_unsuccessful(test_client): + email = "test@example.com" + response = test_client.post( + "/signup", data=dict(name="Test", surname="Foo", email=email, password="test1234"), follow_redirects=True + ) + assert response.request.path == url_for("auth.show_signup_form"), "Signup was unsuccessful" + assert f"Email {email} in use".encode("utf-8") in response.data + + +def test_signup_user_successful(test_client): + response = test_client.post( + "/signup", + data=dict(name="Foo", surname="Example", email="foo@example.com", password="foo1234"), + follow_redirects=True, + ) + assert response.request.path == url_for("public.index"), "Signup was unsuccessful" diff --git a/app/blueprints/dataset/repositories.py b/app/blueprints/dataset/repositories.py index c1d4aa897..b1b54d996 100644 --- a/app/blueprints/dataset/repositories.py +++ b/app/blueprints/dataset/repositories.py @@ -1,5 +1,9 @@ +import re +import unidecode from typing import Optional +from sqlalchemy import or_, any_ + from app.blueprints.dataset.models import ( Author, DSDownloadRecord, @@ -10,6 +14,7 @@ FeatureModel, File, FileDownloadRecord, + PublicationType, ) from core.repositories.BaseRepository import BaseRepository @@ -44,20 +49,70 @@ def __init__(self): def get_synchronized(self, current_user_id: int) -> DataSet: return ( self.model.query.join(DSMetaData) - .filter( - DataSet.user_id == current_user_id, DSMetaData.dataset_doi.isnot(None) - ) - .order_by(DataSet.created_at.desc()) + .filter(DataSet.user_id == current_user_id, DSMetaData.dataset_doi.isnot(None)) + .order_by(self.model.created_at.desc()) .all() ) def get_unsynchronized(self, current_user_id: int) -> DataSet: return ( self.model.query.join(DSMetaData) - .filter( - DataSet.user_id == current_user_id, DSMetaData.dataset_doi.is_(None) - ) - .order_by(DataSet.created_at.desc()) + .filter(DataSet.user_id == current_user_id, DSMetaData.dataset_doi.is_(None)) + .order_by(self.model.created_at.desc()) + .all() + ) + + def filter(self, query="", sorting="newest", publication_type="any", tags=[], **kwargs): + # Normalize and remove unwanted characters + normalized_query = unidecode.unidecode(query).lower() + cleaned_query = re.sub(r'[,.":\'()\[\]^;!¡¿?]', "", normalized_query) + + filters = [] + for word in cleaned_query.split(): + filters.append(DSMetaData.title.ilike(f"%{word}%")) + filters.append(DSMetaData.description.ilike(f"%{word}%")) + filters.append(Author.name.ilike(f"%{word}%")) + filters.append(Author.affiliation.ilike(f"%{word}%")) + filters.append(Author.orcid.ilike(f"%{word}%")) + filters.append(FMMetaData.uvl_filename.ilike(f"%{word}%")) + filters.append(FMMetaData.title.ilike(f"%{word}%")) + filters.append(FMMetaData.description.ilike(f"%{word}%")) + filters.append(FMMetaData.publication_type.ilike(f"%{word}%")) + filters.append(FMMetaData.publication_doi.ilike(f"%{word}%")) + filters.append(FMMetaData.tags.ilike(f"%{word}%")) + filters.append(DSMetaData.tags.ilike(f"%{word}%")) + + datasets = ( + self.model.query.join(DSMetaData).join(Author).join(FeatureModel).join(FMMetaData).filter(or_(*filters)) + ) + + if publication_type != "any": + matching_type = None + for member in PublicationType: + if member.value.lower() == publication_type: + matching_type = member + break + + if matching_type is not None: + datasets = datasets.filter(DSMetaData.publication_type == matching_type.name) + + if tags: + datasets = datasets.filter(DSMetaData.tags.ilike(any_(f"%{tag}%" for tag in tags))) + + # Order by created_at + if sorting == "oldest": + datasets = datasets.order_by(self.model.created_at.asc()) + else: + datasets = datasets.order_by(self.model.created_at.desc()) + + return datasets.all() + + def latest_synchronized(self): + return ( + self.model.query.join(DSMetaData) + .filter(DSMetaData.dataset_doi.isnot(None)) + .order_by(self.model.created_at.desc()) + .limit(5) .all() ) diff --git a/app/blueprints/dataset/routes.py b/app/blueprints/dataset/routes.py index 1cf85675b..2f9385e74 100644 --- a/app/blueprints/dataset/routes.py +++ b/app/blueprints/dataset/routes.py @@ -62,6 +62,7 @@ def create_dataset(): data = json.loads(response_data) except Exception: data = {} + zenodo_response_json = {} if data.get("conceptrecid"): deposition_id = data.get("id") diff --git a/app/blueprints/dataset/services.py b/app/blueprints/dataset/services.py new file mode 100644 index 000000000..22745616c --- /dev/null +++ b/app/blueprints/dataset/services.py @@ -0,0 +1,7 @@ +from app.blueprints.dataset.repositories import DataSetRepository +from core.services.BaseService import BaseService + + +class DataSetService(BaseService): + def __init__(self): + super().__init__(DataSetRepository()) diff --git a/app/blueprints/explore/routes.py b/app/blueprints/explore/routes.py index 34a5944c5..e7dce8fd6 100644 --- a/app/blueprints/explore/routes.py +++ b/app/blueprints/explore/routes.py @@ -1,13 +1,8 @@ -import re - -import unidecode - from flask import render_template, request, jsonify -from sqlalchemy import or_, any_ from app.blueprints.explore import explore_bp from app.blueprints.explore.forms import ExploreForm -from app.blueprints.dataset.models import DataSet, DSMetaData, Author, FeatureModel, FMMetaData, PublicationType +from app.blueprints.dataset.repositories import DataSetRepository @explore_bp.route('/explore', methods=['GET', 'POST']) @@ -16,62 +11,8 @@ def index(): query = request.args.get('query', '') form = ExploreForm() return render_template('explore/index.html', form=form, query=query) - elif request.method == 'POST': - criteria = request.get_json() - query = criteria.get('query', '') - order = criteria.get('sorting', 'newest') - publication_type = criteria.get('publication_type', 'any') - tags = criteria.get('tags', []) - - # Normalize and remove unwanted characters - normalized_query = unidecode.unidecode(query).lower() - cleaned_query = re.sub(r'[,.":\'()\[\]^;!¡¿?]', '', normalized_query) - query_words = cleaned_query.split() - - filters = [] - for word in query_words: - filters.append(DSMetaData.title.ilike(f'%{word}%')) - filters.append(DSMetaData.description.ilike(f'%{word}%')) - filters.append(Author.name.ilike(f'%{word}%')) - filters.append(Author.affiliation.ilike(f'%{word}%')) - filters.append(Author.orcid.ilike(f'%{word}%')) - filters.append(FMMetaData.uvl_filename.ilike(f'%{word}%')) - filters.append(FMMetaData.title.ilike(f'%{word}%')) - filters.append(FMMetaData.description.ilike(f'%{word}%')) - filters.append(FMMetaData.publication_type.ilike(f'%{word}%')) - filters.append(FMMetaData.publication_doi.ilike(f'%{word}%')) - filters.append(FMMetaData.tags.ilike(f'%{word}%')) - filters.append(DSMetaData.tags.ilike(f'%{word}%')) - - datasets = DataSet.query \ - .join(DSMetaData) \ - .join(Author) \ - .join(FeatureModel) \ - .join(FMMetaData) - - if publication_type != 'any': - matching_type = None - for member in PublicationType: - if member.value.lower() == publication_type: - matching_type = member - break - if matching_type is not None: - datasets = datasets.filter(DSMetaData.publication_type == matching_type.name) - - datasets = datasets.filter(or_(*filters)) - - if tags: - datasets = datasets.filter(DSMetaData.tags.ilike(any_(f'%{tag}%' for tag in tags))) - - # Order by created_at - if order == 'oldest': - datasets = datasets.order_by(DataSet.created_at.asc()) - else: - datasets = datasets.order_by(DataSet.created_at.desc()) - - datasets = datasets.all() - - dataset_dicts = [dataset.to_dict() for dataset in datasets] - - return jsonify(dataset_dicts) + if request.method == 'POST': + criteria = request.get_json() + datasets = DataSetRepository().filter(**criteria) + return jsonify([dataset.to_dict() for dataset in datasets]) diff --git a/app/blueprints/profile/routes.py b/app/blueprints/profile/routes.py index 83eb57c04..5ab2b218c 100644 --- a/app/blueprints/profile/routes.py +++ b/app/blueprints/profile/routes.py @@ -1,22 +1,25 @@ -from flask import request, render_template +from flask import render_template, redirect, url_for, request from flask_login import login_required +from app import get_authenticated_user_profile from app.blueprints.profile import profile_bp from app.blueprints.profile.forms import UserProfileForm - -from app import get_authenticated_user_profile from app.blueprints.profile.services import UserProfileService -@profile_bp.route('/profile/edit', methods=['GET', 'POST']) +@profile_bp.route("/profile/edit", methods=["GET", "POST"]) @login_required def edit_profile(): - form = UserProfileForm() - if request.method == 'POST': + profile = get_authenticated_user_profile() + if not profile: + return redirect(url_for("public.index")) + form = UserProfileForm() + if request.method == "POST": service = UserProfileService() - result, errors = service.update_profile(get_authenticated_user_profile().id, form) - return service.handle_service_response(result, errors, 'profile.edit_profile', 'Profile updated successfully', - 'profile/edit.html', form) + result, errors = service.update_profile(profile.id, form) + return service.handle_service_response( + result, errors, "profile.edit_profile", "Profile updated successfully", "profile/edit.html", form + ) - return render_template('profile/edit.html', form=form) + return render_template("profile/edit.html", form=form) diff --git a/app/blueprints/profile/services.py b/app/blueprints/profile/services.py index 4c66b2cc0..41dcaa6f6 100644 --- a/app/blueprints/profile/services.py +++ b/app/blueprints/profile/services.py @@ -10,5 +10,5 @@ def update_profile(self, user_profile_id, form): if form.validate(): updated_instance = self.update(user_profile_id, **form.data) return updated_instance, None - else: - return None, form.errors + + return None, form.errors diff --git a/app/blueprints/profile/tests/test_unit.py b/app/blueprints/profile/tests/test_unit.py index 7f6bf9c00..4215d89a7 100644 --- a/app/blueprints/profile/tests/test_unit.py +++ b/app/blueprints/profile/tests/test_unit.py @@ -1,18 +1,25 @@ import pytest +from app import db from app.blueprints.conftest import login, logout +from app.blueprints.auth.models import User +from app.blueprints.profile.models import UserProfile -@pytest.fixture(scope='module') +@pytest.fixture(scope="module") def test_client(test_client): """ Extends the test_client fixture to add additional specific data for module testing. for module testing (por example, new users) """ with test_client.application.app_context(): - # Add HERE new elements to the database that you want to exist in the test context. - # DO NOT FORGET to use db.session.add() and db.session.commit() to save the data. - pass + user_test = User(email='user@example.com', password='test1234') + db.session.add(user_test) + db.session.commit() + + profile = UserProfile(user_id=user_test.id, name="Name", surname="Surname") + db.session.add(profile) + db.session.commit() yield test_client @@ -21,10 +28,10 @@ def test_edit_profile_page_get(test_client): """ Tests access to the profile editing page via a GET request. """ - login_response = login(test_client, 'test@example.com', 'test1234') + login_response = login(test_client, "user@example.com", "test1234") assert login_response.status_code == 200, "Login was unsuccessful." - response = test_client.get('/profile/edit') + response = test_client.get("/profile/edit") assert response.status_code == 200, "The profile editing page could not be accessed." assert b"Edit profile" in response.data, "The expected content is not present on the page" diff --git a/app/blueprints/public/routes.py b/app/blueprints/public/routes.py index 40c3a7109..dc29debe9 100644 --- a/app/blueprints/public/routes.py +++ b/app/blueprints/public/routes.py @@ -1,34 +1,28 @@ import logging +from flask import render_template from flask_login import login_required -import app - -from flask import render_template from app.blueprints.public import public_bp -from ..dataset.models import DataSet, DSMetaData +from app.blueprints.dataset.repositories import DataSetRepository, FeatureModelRepository logger = logging.getLogger(__name__) +dataset_repository = DataSetRepository() @public_bp.route("/") def index(): - logger.info('Access index') - - latest_datasets = DataSet.query.join(DSMetaData).filter( - DSMetaData.dataset_doi.isnot(None) - ).order_by(DataSet.created_at.desc()).limit(5).all() - - datasets_counter = app.datasets_counter() - feature_models_counter = app.feature_models_counter() + logger.info("Access index") - return render_template("public/index.html", - datasets=latest_datasets, - datasets_counter=datasets_counter, - feature_models_counter=feature_models_counter) + return render_template( + "public/index.html", + datasets=dataset_repository.latest_synchronized(), + datasets_counter=dataset_repository.count(), + feature_models_counter=FeatureModelRepository().count(), + ) -@public_bp.route('/secret') +@public_bp.route("/secret") @login_required def secret(): return "Esto es secreto!" diff --git a/app/blueprints/pytest.ini b/app/blueprints/pytest.ini index b0e5a945f..c24fe5bb9 100644 --- a/app/blueprints/pytest.ini +++ b/app/blueprints/pytest.ini @@ -1,3 +1,3 @@ [pytest] filterwarnings = - ignore::DeprecationWarning \ No newline at end of file + ignore::DeprecationWarning diff --git a/app/blueprints/zenodo/services.py b/app/blueprints/zenodo/services.py index 4b8308725..3c8d4ca34 100644 --- a/app/blueprints/zenodo/services.py +++ b/app/blueprints/zenodo/services.py @@ -15,9 +15,7 @@ class ZenodoService(BaseService): - ZENODO_API_URL = os.getenv( - "ZENODO_API_URL", "https://sandbox.zenodo.org/api/deposit/depositions" - ) + ZENODO_API_URL = os.getenv("ZENODO_API_URL", "https://sandbox.zenodo.org/api/deposit/depositions") ZENODO_ACCESS_TOKEN = os.getenv("ZENODO_ACCESS_TOKEN") def __init__(self): @@ -32,14 +30,13 @@ def test_connection(self) -> bool: Returns: bool: True if the connection is successful, False otherwise. """ - response = requests.get( - self.ZENODO_API_URL, params=self.params, headers=self.headers - ) + response = requests.get(self.ZENODO_API_URL, params=self.params, headers=self.headers) return response.status_code == 200 def test_full_connection(self) -> Response: """ - Test the connection with Zenodo by creating a deposition, uploading an empty test file, and deleting the deposition. + Test the connection with Zenodo by creating a deposition, uploading an empty test file, and deleting the + deposition. Returns: bool: True if the connection, upload, and deletion are successful, False otherwise. @@ -64,9 +61,7 @@ def test_full_connection(self) -> Response: } } - response = requests.post( - self.ZENODO_API_URL, json=data, params=self.params, headers=self.headers - ) + response = requests.post(self.ZENODO_API_URL, json=data, params=self.params, headers=self.headers) if response.status_code != 201: return jsonify( @@ -83,20 +78,14 @@ def test_full_connection(self) -> Response: file_path = os.path.join(current_app.root_path, "test_file.txt") files = {"file": open(file_path, "rb")} publish_url = f"{self.ZENODO_API_URL}/{deposition_id}/files" - response = requests.post( - publish_url, params=self.params, data=data, files=files - ) + response = requests.post(publish_url, params=self.params, data=data, files=files) if response.status_code != 201: - messages.append( - f"Failed to upload test file to Zenodo. Response code: {response.status_code}" - ) + messages.append(f"Failed to upload test file to Zenodo. Response code: {response.status_code}") success = False # Step 3: Delete the deposition - response = requests.delete( - f"{self.ZENODO_API_URL}/{deposition_id}", params=self.params - ) + response = requests.delete(f"{self.ZENODO_API_URL}/{deposition_id}", params=self.params) if os.path.exists(file_path): os.remove(file_path) @@ -110,9 +99,7 @@ def get_all_depositions(self) -> dict: Returns: dict: The response in JSON format with the depositions. """ - response = requests.get( - self.ZENODO_API_URL, params=self.params, headers=self.headers - ) + response = requests.get(self.ZENODO_API_URL, params=self.params, headers=self.headers) if response.status_code != 200: raise Exception("Failed to get depositions") return response.json() @@ -129,47 +116,37 @@ def create_new_deposition(self, dataset: DataSet) -> dict: """ metadata = { "title": dataset.ds_meta_data.title, - "upload_type": "dataset" - if dataset.ds_meta_data.publication_type.value == "none" - else "publication", - "publication_type": dataset.ds_meta_data.publication_type.value - if dataset.ds_meta_data.publication_type.value != "none" - else None, + "upload_type": "dataset" if dataset.ds_meta_data.publication_type.value == "none" else "publication", + "publication_type": ( + dataset.ds_meta_data.publication_type.value + if dataset.ds_meta_data.publication_type.value != "none" + else None + ), "description": dataset.ds_meta_data.description, "creators": [ { "name": author.name, - **( - {"affiliation": author.affiliation} - if author.affiliation - else {} - ), + **({"affiliation": author.affiliation} if author.affiliation else {}), **({"orcid": author.orcid} if author.orcid else {}), } for author in dataset.ds_meta_data.authors ], - "keywords": ["uvlhub"] - if not dataset.ds_meta_data.tags - else dataset.ds_meta_data.tags.split(", ") + ["uvlhub"], + "keywords": ( + ["uvlhub"] if not dataset.ds_meta_data.tags else dataset.ds_meta_data.tags.split(", ") + ["uvlhub"] + ), "access_right": "open", "license": "CC-BY-4.0", } data = {"metadata": metadata} - response = requests.post( - self.ZENODO_API_URL, params=self.params, json=data, headers=self.headers - ) + response = requests.post(self.ZENODO_API_URL, params=self.params, json=data, headers=self.headers) if response.status_code != 201: - error_message = ( - f"Failed to create deposition. Error details: {response.json()}" - ) + error_message = f"Failed to create deposition. Error details: {response.json()}" raise Exception(error_message) return response.json() - def upload_file( - self, deposition_id: int, feature_model: FeatureModel, user=None - ) -> dict: + def upload_file(self, deposition_id: int, feature_model: FeatureModel, user=None) -> dict: """ Upload a file to a deposition in Zenodo. @@ -184,15 +161,11 @@ def upload_file( uvl_filename = feature_model.fm_meta_data.uvl_filename data = {"name": uvl_filename} user_id = current_user.id if user is None else user.id - file_path = os.path.join( - app.upload_folder_name(), "temp", str(user_id), uvl_filename - ) + file_path = os.path.join(app.upload_folder_name(), "temp", str(user_id), uvl_filename) files = {"file": open(file_path, "rb")} publish_url = f"{self.ZENODO_API_URL}/{deposition_id}/files" - response = requests.post( - publish_url, params=self.params, data=data, files=files - ) + response = requests.post(publish_url, params=self.params, data=data, files=files) if response.status_code != 201: error_message = f"Failed to upload files. Error details: {response.json()}" raise Exception(error_message) @@ -225,9 +198,7 @@ def get_deposition(self, deposition_id: int) -> dict: dict: The response in JSON format with the details of the deposition. """ deposition_url = f"{self.ZENODO_API_URL}/{deposition_id}" - response = requests.get( - deposition_url, params=self.params, headers=self.headers - ) + response = requests.get(deposition_url, params=self.params, headers=self.headers) if response.status_code != 200: raise Exception("Failed to get deposition") return response.json() diff --git a/core/repositories/BaseRepository.py b/core/repositories/BaseRepository.py index 95029c48a..1aa459695 100644 --- a/core/repositories/BaseRepository.py +++ b/core/repositories/BaseRepository.py @@ -38,3 +38,6 @@ def delete(self, id: int) -> bool: app.db.session.commit() return True return False + + def count(self) -> int: + return self.model.query.count() From 184d2811f4c8993789ae2f4212b0d655ce861bd9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?V=C3=ADctor=20Ram=C3=ADrez=20de=20la=20Corte?= Date: Tue, 2 Jul 2024 13:51:01 +0200 Subject: [PATCH 3/7] feat: create service for auth and implement rollback when create user and profile --- app/blueprints/auth/repositories.py | 10 +-- app/blueprints/auth/routes.py | 6 +- app/blueprints/auth/services.py | 27 ++++++++ app/blueprints/auth/tests/test_unit.py | 36 +++++++++++ app/blueprints/conftest.py | 14 ++++ core/repositories/BaseRepository.py | 90 ++++++++++++++------------ core/services/BaseService.py | 62 ++++++++++-------- 7 files changed, 168 insertions(+), 77 deletions(-) diff --git a/app/blueprints/auth/repositories.py b/app/blueprints/auth/repositories.py index fadd6f76c..fa247f900 100644 --- a/app/blueprints/auth/repositories.py +++ b/app/blueprints/auth/repositories.py @@ -1,4 +1,3 @@ -import app from app.blueprints.auth.models import User from core.repositories.BaseRepository import BaseRepository @@ -7,12 +6,15 @@ class UserRepository(BaseRepository): def __init__(self): super().__init__(User) - def create(self, **kwargs): + def create(self, commit: bool = True, **kwargs): password = kwargs.pop("password") instance = self.model(**kwargs) instance.set_password(password) - app.db.session.add(instance) - app.db.session.commit() + self.session.add(instance) + if commit: + self.session.commit() + else: + self.session.flush() return instance def get_by_email(self, email: str): diff --git a/app/blueprints/auth/routes.py b/app/blueprints/auth/routes.py index 5d4d9a903..e42076ebe 100644 --- a/app/blueprints/auth/routes.py +++ b/app/blueprints/auth/routes.py @@ -22,8 +22,10 @@ def show_signup_form(): if not authentication_service.is_email_available(email): return render_template("auth/signup_form.html", form=form, error=f'Email {email} in use') - user = authentication_service.create(email=email, password=form.password.data) - user_profile_service.create(name=form.name.data, surname=form.surname.data, user_id=user.id) + try: + user = authentication_service.create_with_profile(**form.data) + except Exception as exc: + return render_template("auth/signup_form.html", form=form, error=f'Error creating user: {exc}') # Log user login_user(user, remember=True) diff --git a/app/blueprints/auth/services.py b/app/blueprints/auth/services.py index 1533ffc27..5be5b5210 100644 --- a/app/blueprints/auth/services.py +++ b/app/blueprints/auth/services.py @@ -1,12 +1,15 @@ from flask_login import login_user +from app import db from app.blueprints.auth.repositories import UserRepository +from app.blueprints.profile.repositories import UserProfileRepository from core.services.BaseService import BaseService class AuthenticationService(BaseService): def __init__(self): super().__init__(UserRepository()) + self.user_profile_repository = UserProfileRepository() def login(self, email, password, remember=True): user = self.repository.get_by_email(email) @@ -17,3 +20,27 @@ def login(self, email, password, remember=True): def is_email_available(self, email: str) -> bool: return self.repository.get_by_email(email) is None + + def create_with_profile(self, **kwargs): + try: + profile_data = { + "name": kwargs.pop("name"), + "surname": kwargs.pop("surname"), + "orcid": kwargs.pop("orcid"), + "affiliation": kwargs.pop("affiliation"), + } + user = self.create(commit=False, **kwargs) + profile_data["user_id"] = user.id + self.user_profile_repository.create(**profile_data) + self.repository.session.commit() + except Exception as exc: + db.session.rollback() + raise exc + return user + + def update_profile(self, user_profile_id, form): + if form.validate(): + updated_instance = self.update(user_profile_id, **form.data) + return updated_instance, None + + return None, form.errors diff --git a/app/blueprints/auth/tests/test_unit.py b/app/blueprints/auth/tests/test_unit.py index 65c3aa8d1..cc98aa78c 100644 --- a/app/blueprints/auth/tests/test_unit.py +++ b/app/blueprints/auth/tests/test_unit.py @@ -1,6 +1,10 @@ import pytest from flask import url_for +from app.blueprints.auth.services import AuthenticationService +from app.blueprints.auth.repositories import UserRepository +from app.blueprints.profile.repositories import UserProfileRepository + @pytest.fixture(scope="module") def test_client(test_client): @@ -70,3 +74,35 @@ def test_signup_user_successful(test_client): follow_redirects=True, ) assert response.request.path == url_for("public.index"), "Signup was unsuccessful" + + +def test_create_user_success(app): + data = { + "name": "Test", + "surname": "Foo", + "email": "test@example.com", + "password": "test1234", + "orcid": "0000000000000000", + "affiliation": "Example" + } + AuthenticationService().create_with_profile(**data) + assert UserRepository().count() == 1 + assert UserProfileRepository().count() == 1 + + +def test_create_user_fail_profile(app): + data = { + "name": None, + "surname": "Foo", + "email": "test@example.com", + "password": "test1234", + "orcid": "0000000000000000", + "affiliation": "Example", + } + try: + AuthenticationService().create_with_profile(**data) + except Exception: + pass + + assert UserProfileRepository().count() == 0 + assert UserRepository().count() == 0 diff --git a/app/blueprints/conftest.py b/app/blueprints/conftest.py index 3731e5fdf..088d5d500 100644 --- a/app/blueprints/conftest.py +++ b/app/blueprints/conftest.py @@ -52,3 +52,17 @@ def logout(test_client): response: Response to GET request to log out. """ return test_client.get('/logout', follow_redirects=True) + + +@pytest.fixture(scope='function') +def app(): + flask_app = create_app('testing') + with flask_app.app_context(): + db.create_all() + """ + The test suite always includes the following user in order to avoid repetition + of its creation + """ + yield app + db.session.remove() + db.drop_all() diff --git a/core/repositories/BaseRepository.py b/core/repositories/BaseRepository.py index 1aa459695..f00892e40 100644 --- a/core/repositories/BaseRepository.py +++ b/core/repositories/BaseRepository.py @@ -1,43 +1,47 @@ -from typing import Generic, NoReturn, Optional, TypeVar, Union - -import app - -T = TypeVar('T') - - -class BaseRepository(Generic[T]): - def __init__(self, model: T): - self.model = model - - def create(self, **kwargs) -> T: - instance: T = self.model(**kwargs) - app.db.session.add(instance) - app.db.session.commit() - return instance - - def get_by_id(self, id: int) -> Optional[T]: - instance: Optional[T] = self.model.query.get(id) - return instance - - def get_or_404(self, id: int) -> Union[T, NoReturn]: - return self.model.query.get_or_404(id) - - def update(self, id: int, **kwargs) -> Optional[T]: - instance: Optional[T] = self.get_by_id(id) - if instance: - for key, value in kwargs.items(): - setattr(instance, key, value) - app.db.session.commit() - return instance - return None - - def delete(self, id: int) -> bool: - instance: Optional[T] = self.get_by_id(id) - if instance: - app.db.session.delete(instance) - app.db.session.commit() - return True - return False - - def count(self) -> int: - return self.model.query.count() +from typing import Generic, NoReturn, Optional, TypeVar, Union + +import app + +T = TypeVar('T') + + +class BaseRepository(Generic[T]): + def __init__(self, model: T): + self.model = model + self.session = app.db.session + + def create(self, commit: bool = True, **kwargs) -> T: + instance: T = self.model(**kwargs) + self.session.add(instance) + if commit: + self.session.commit() + else: + self.session.flush() + return instance + + def get_by_id(self, id: int) -> Optional[T]: + instance: Optional[T] = self.model.query.get(id) + return instance + + def get_or_404(self, id: int) -> Union[T, NoReturn]: + return self.model.query.get_or_404(id) + + def update(self, id: int, **kwargs) -> Optional[T]: + instance: Optional[T] = self.get_by_id(id) + if instance: + for key, value in kwargs.items(): + setattr(instance, key, value) + self.session.commit() + return instance + return None + + def delete(self, id: int) -> bool: + instance: Optional[T] = self.get_by_id(id) + if instance: + self.session.delete(instance) + self.session.commit() + return True + return False + + def count(self) -> int: + return self.model.query.count() diff --git a/core/services/BaseService.py b/core/services/BaseService.py index 3ff8d6472..942dca6b8 100644 --- a/core/services/BaseService.py +++ b/core/services/BaseService.py @@ -1,28 +1,34 @@ -from flask import flash, redirect, url_for, render_template - - -class BaseService: - def __init__(self, repository): - self.repository = repository - - def create(self, **kwargs): - return self.repository.create(**kwargs) - - def get_by_id(self, id): - return self.repository.get_by_id(id) - - def update(self, id, **kwargs): - return self.repository.update(id, **kwargs) - - def delete(self, id): - return self.repository.delete(id) - - def handle_service_response(self, result, errors, success_url_redirect, success_msg, error_template, form): - if result: - flash(success_msg, 'success') - return redirect(url_for(success_url_redirect)) - else: - for error_field, error_messages in errors.items(): - for error_message in error_messages: - flash(f'{error_field}: {error_message}', 'error') - return render_template(error_template, form=form) +from flask import flash, redirect, url_for, render_template + + +class BaseService: + def __init__(self, repository): + self.repository = repository + + def create(self, **kwargs): + return self.repository.create(**kwargs) + + def count(self) -> int: + return self.repository.count() + + def get_by_id(self, id): + return self.repository.get_by_id(id) + + def get_or_404(self, id): + return self.repository.get_or_404(id) + + def update(self, id, **kwargs): + return self.repository.update(id, **kwargs) + + def delete(self, id): + return self.repository.delete(id) + + def handle_service_response(self, result, errors, success_url_redirect, success_msg, error_template, form): + if result: + flash(success_msg, 'success') + return redirect(url_for(success_url_redirect)) + else: + for error_field, error_messages in errors.items(): + for error_message in error_messages: + flash(f'{error_field}: {error_message}', 'error') + return render_template(error_template, form=form) From c4109db4606f9938647f6247352a23e058e489a7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?V=C3=ADctor=20Ram=C3=ADrez=20de=20la=20Corte?= Date: Tue, 2 Jul 2024 13:55:36 +0200 Subject: [PATCH 4/7] refactor: create services and use instead repository --- app/blueprints/dataset/routes.py | 67 ++++++++------- app/blueprints/dataset/services.py | 85 ++++++++++++++++++- .../templates/dataset/upload_dataset.html | 2 +- app/blueprints/explore/routes.py | 4 +- app/blueprints/public/routes.py | 10 +-- 5 files changed, 127 insertions(+), 41 deletions(-) diff --git a/app/blueprints/dataset/routes.py b/app/blueprints/dataset/routes.py index 2f9385e74..4fd2647cb 100644 --- a/app/blueprints/dataset/routes.py +++ b/app/blueprints/dataset/routes.py @@ -22,20 +22,23 @@ from app.blueprints.dataset.forms import DataSetForm from app.blueprints.dataset.models import DataSet, PublicationType from app.blueprints.dataset import dataset_bp -from app.blueprints.dataset.repositories import ( - AuthorRepository, - DSDownloadRecordRepository, - DSMetaDataRepository, - DSViewRecordRepository, - DataSetRepository, - FMMetaDataRepository, - FeatureModelRepository, - FileRepository, - FileDownloadRecordRepository, +from app.blueprints.dataset.services import ( + AuthorService, + DSDownloadRecordService, + DSMetaDataService, + DSViewRecordService, + DataSetService, + FMMetaDataService, + FeatureModelService, + FileService, + FileDownloadRecordService, ) from app.blueprints.zenodo.services import ZenodoService +dataset_service = DataSetService() +author_service = AuthorService() +dsmetadata_service = DSMetaDataService() zenodo_service = ZenodoService() @@ -68,7 +71,7 @@ def create_dataset(): deposition_id = data.get("id") # update dataset with deposition id in Zenodo - DSMetaDataRepository().update( + dsmetadata_service.update( dataset.ds_meta_data_id, deposition_id=deposition_id ) @@ -87,7 +90,7 @@ def create_dataset(): # update DOI deposition_doi = zenodo_service.get_doi(deposition_id) - DSMetaDataRepository().update( + dsmetadata_service.update( dataset.ds_meta_data_id, dataset_doi=deposition_doi ) except Exception: @@ -125,8 +128,8 @@ def create_dataset(): def list_dataset(): return render_template( "dataset/list_datasets.html", - datasets=DataSetRepository().get_synchronized(current_user.id), - local_datasets=DataSetRepository().get_unsynchronized(current_user.id), + datasets=dataset_service.get_synchronized(current_user.id), + local_datasets=dataset_service.get_unsynchronized(current_user.id), ) @@ -138,7 +141,7 @@ def create_dataset_in_db(basic_info_data): "publication_doi": basic_info_data["publication_doi"][0], "tags": basic_info_data["tags"][0], } - ds_meta_data = DSMetaDataRepository().create(**ds_data) + ds_meta_data = dsmetadata_service.create(**ds_data) # create dataset metadata authors # I always add myself @@ -152,7 +155,7 @@ def create_dataset_in_db(basic_info_data): else None, "ds_meta_data_id": ds_meta_data.id, } - AuthorRepository().create(**author_data) + author_service.create(**author_data) # how many authors are there? if "author_name" in basic_info_data: @@ -164,10 +167,10 @@ def create_dataset_in_db(basic_info_data): "orcid": basic_info_data["author_orcid"][i], "ds_meta_data_id": ds_meta_data.id, } - AuthorRepository().create(**extra_author) + author_service.create(**extra_author) # create dataset - return DataSetRepository().create( + return dataset_service.create( user_id=current_user.id, ds_meta_data_id=ds_meta_data.id ) @@ -179,7 +182,7 @@ def create_feature_models_in_db(dataset: DataSet, uploaded_models_data: dict): uvl_filename = uploaded_models_data["uvl_filename"][i] # create feature model metadata - feature_model_metadata = FMMetaDataRepository().create( + feature_model_metadata = FMMetaDataService().create( uvl_filename=uvl_filename, title=uploaded_models_data["title"][i], description=uploaded_models_data["description"][i], @@ -192,7 +195,7 @@ def create_feature_models_in_db(dataset: DataSet, uploaded_models_data: dict): ) # create feature model - feature_model = FeatureModelRepository().create( + feature_model = FeatureModelService().create( data_set_id=dataset.id, fm_meta_data_id=feature_model_metadata.id, ) @@ -201,7 +204,7 @@ def create_feature_models_in_db(dataset: DataSet, uploaded_models_data: dict): for idx, author_name in enumerate( uploaded_models_data.get(f"author_name_{uvl_identifier}", []) ): - AuthorRepository().create( + author_service.create( name=author_name, affiliation=uploaded_models_data[ f"author_affiliation_{uvl_identifier}" @@ -217,7 +220,7 @@ def create_feature_models_in_db(dataset: DataSet, uploaded_models_data: dict): ) checksum, size = calculate_checksum_and_size(file_path) - FileRepository().create( + FileService().create( name=uvl_filename, checksum=checksum, size=size, @@ -237,8 +240,8 @@ def calculate_checksum_and_size(file_path): return hash_md5, file_size -def move_feature_models(dataset_id, feature_models, user=None): - user_id = current_user.id if user is None else user.id +def move_feature_models(dataset_id, feature_models): + user_id = current_user.id source_dir = f"uploads/temp/{user_id}/" dest_dir = f"uploads/user_{user_id}/dataset_{dataset_id}/" @@ -313,7 +316,7 @@ def delete(): @dataset_bp.route("/dataset/download/", methods=["GET"]) def download_dataset(dataset_id): - dataset = DataSetRepository().get_or_404(dataset_id) + dataset = dataset_service.get_or_404(dataset_id) file_path = f"uploads/user_{dataset.user_id}/dataset_{dataset.id}/" @@ -358,7 +361,7 @@ def download_dataset(dataset_id): ) # Record the download in your database - DSDownloadRecordRepository().create( + DSDownloadRecordService().create( user_id=current_user.id if current_user.is_authenticated else None, dataset_id=dataset_id, download_date=datetime.utcnow(), @@ -370,7 +373,7 @@ def download_dataset(dataset_id): @dataset_bp.route("/dataset/view/", methods=["GET"]) def view_dataset(dataset_id): - dataset = DataSetRepository().get_or_404(dataset_id) + dataset = dataset_service.get_or_404(dataset_id) # Get the cookie from the request or generate a new one if it does not exist user_cookie = request.cookies.get("view_cookie") @@ -378,7 +381,7 @@ def view_dataset(dataset_id): user_cookie = str(uuid.uuid4()) # Record the view in your database - DSViewRecordRepository().create( + DSViewRecordService().create( user_id=current_user.id if current_user.is_authenticated else None, dataset_id=dataset_id, view_date=datetime.utcnow(), @@ -394,7 +397,7 @@ def view_dataset(dataset_id): @dataset_bp.route("/file/download/", methods=["GET"]) def download_file(file_id): - file = FileRepository().get_or_404(file_id) + file = FileService().get_or_404(file_id) filename = file.name directory_path = f"uploads/user_{file.feature_model.data_set.user_id}/dataset_{file.feature_model.data_set_id}/" @@ -407,7 +410,7 @@ def download_file(file_id): user_cookie = str(uuid.uuid4()) # Record the download in your database - FileDownloadRecordRepository().create( + FileDownloadRecordService().create( user_id=current_user.id if current_user.is_authenticated else None, file_id=file_id, download_date=datetime.utcnow(), @@ -426,7 +429,7 @@ def download_file(file_id): @dataset_bp.route("/doi//", methods=["GET"]) def subdomain_index(doi): # Busca el dataset por DOI - ds_meta_data = DSMetaDataRepository().filter_by_doi(doi) + ds_meta_data = dsmetadata_service.filter_by_doi(doi) if not ds_meta_data: abort(404) @@ -436,7 +439,7 @@ def subdomain_index(doi): user_cookie = request.cookies.get("view_cookie", str(uuid.uuid4())) # Registra la vista del dataset - DSViewRecordRepository().create( + DSViewRecordService().create( user_id=current_user.id if current_user.is_authenticated else None, dataset_id=dataset_id, view_date=datetime.utcnow(), diff --git a/app/blueprints/dataset/services.py b/app/blueprints/dataset/services.py index 22745616c..de9db8628 100644 --- a/app/blueprints/dataset/services.py +++ b/app/blueprints/dataset/services.py @@ -1,7 +1,90 @@ -from app.blueprints.dataset.repositories import DataSetRepository +from typing import Optional + +from app.blueprints.dataset.models import DataSet, DSMetaData +from app.blueprints.dataset.repositories import ( + AuthorRepository, + DSDownloadRecordRepository, + DSMetaDataRepository, + DSViewRecordRepository, + DataSetRepository, + FMMetaDataRepository, + FeatureModelRepository, + FileRepository, + FileDownloadRecordRepository, +) from core.services.BaseService import BaseService class DataSetService(BaseService): def __init__(self): super().__init__(DataSetRepository()) + self.feature_model_service = FeatureModelService() + self.author_repository = AuthorRepository() + self.dsmetadata_repository = DSMetaDataRepository() + + def get_synchronized(self, current_user_id: int) -> DataSet: + return self.repository.get_synchronized(current_user_id) + + def get_unsynchronized(self, current_user_id: int) -> DataSet: + return self.repository.get_unsynchronized(current_user_id) + + def latest_synchronized(self): + return self.repository.latest_synchronized() + + def filter(self, query="", sorting="newest", publication_type="any", tags=[], **kwargs): + return self.repository.filter(query, sorting, publication_type, tags, **kwargs) + + def count_feature_models(self): + return self.feature_model_service.count() + + def count_authors(self) -> int: + return self.author_repository.count() + + def count_dsmetadata(self) -> int: + return self.dsmetadata_repository.count() + + +class FeatureModelService(BaseService): + def __init__(self): + super().__init__(FeatureModelRepository()) + + +class AuthorService(BaseService): + def __init__(self): + super().__init__(AuthorRepository()) + + +class DSDownloadRecordService(BaseService): + def __init__(self): + super().__init__(DSDownloadRecordRepository()) + + +class DSMetaDataService(BaseService): + def __init__(self): + super().__init__(DSMetaDataRepository()) + + def update(self, id, **kwargs): + return self.repository.update(id, **kwargs) + + def filter_by_doi(self, doi: str) -> Optional[DSMetaData]: + return self.repository.filter_by_doi(doi) + + +class DSViewRecordService(BaseService): + def __init__(self): + super().__init__(DSViewRecordRepository()) + + +class FMMetaDataService(BaseService): + def __init__(self): + super().__init__(FMMetaDataRepository()) + + +class FileService(BaseService): + def __init__(self): + super().__init__(FileRepository()) + + +class FileDownloadRecordService(BaseService): + def __init__(self): + super().__init__(FileDownloadRecordRepository()) diff --git a/app/blueprints/dataset/templates/dataset/upload_dataset.html b/app/blueprints/dataset/templates/dataset/upload_dataset.html index a38399087..23313ba6e 100644 --- a/app/blueprints/dataset/templates/dataset/upload_dataset.html +++ b/app/blueprints/dataset/templates/dataset/upload_dataset.html @@ -779,4 +779,4 @@

{% endblock %} -{% endblock %} \ No newline at end of file +{% endblock %} diff --git a/app/blueprints/explore/routes.py b/app/blueprints/explore/routes.py index e7dce8fd6..fb38416e9 100644 --- a/app/blueprints/explore/routes.py +++ b/app/blueprints/explore/routes.py @@ -2,7 +2,7 @@ from app.blueprints.explore import explore_bp from app.blueprints.explore.forms import ExploreForm -from app.blueprints.dataset.repositories import DataSetRepository +from app.blueprints.dataset.services import DataSetService @explore_bp.route('/explore', methods=['GET', 'POST']) @@ -14,5 +14,5 @@ def index(): if request.method == 'POST': criteria = request.get_json() - datasets = DataSetRepository().filter(**criteria) + datasets = DataSetService().filter(**criteria) return jsonify([dataset.to_dict() for dataset in datasets]) diff --git a/app/blueprints/public/routes.py b/app/blueprints/public/routes.py index dc29debe9..4695aa015 100644 --- a/app/blueprints/public/routes.py +++ b/app/blueprints/public/routes.py @@ -4,21 +4,21 @@ from flask_login import login_required from app.blueprints.public import public_bp -from app.blueprints.dataset.repositories import DataSetRepository, FeatureModelRepository +from app.blueprints.dataset.services import DataSetService logger = logging.getLogger(__name__) -dataset_repository = DataSetRepository() @public_bp.route("/") def index(): logger.info("Access index") + dataset_service = DataSetService() return render_template( "public/index.html", - datasets=dataset_repository.latest_synchronized(), - datasets_counter=dataset_repository.count(), - feature_models_counter=FeatureModelRepository().count(), + datasets=dataset_service.latest_synchronized(), + datasets_counter=dataset_service.count(), + feature_models_counter=dataset_service.count_feature_models(), ) From 86943af3cdd8f747f718ad7cb940290be313c1df Mon Sep 17 00:00:00 2001 From: David Romero Date: Wed, 3 Jul 2024 13:10:27 +0200 Subject: [PATCH 5/7] fix: Fix coverage --- rosemary/commands/coverage.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rosemary/commands/coverage.py b/rosemary/commands/coverage.py index 3d58671f5..6aaaf82ce 100644 --- a/rosemary/commands/coverage.py +++ b/rosemary/commands/coverage.py @@ -19,7 +19,7 @@ def coverage(module_name, html): else: click.echo("Running coverage for all modules...") - coverage_cmd = ['pytest', '--cov=' + test_path, test_path] + coverage_cmd = ['pytest', '--ignore-glob=*selenium*','--cov=' + test_path, test_path] if html: coverage_cmd.extend(['--cov-report', 'html']) From 55c53c8548976e4cae6a9911948b3832b42c41dc Mon Sep 17 00:00:00 2001 From: David Romero Date: Wed, 3 Jul 2024 15:50:34 +0200 Subject: [PATCH 6/7] fix: Fix dataset routes --- app/blueprints/dataset/routes.py | 119 ++++++++++++++++++++++++++----- 1 file changed, 101 insertions(+), 18 deletions(-) diff --git a/app/blueprints/dataset/routes.py b/app/blueprints/dataset/routes.py index 3ed2deae2..38cb14e47 100644 --- a/app/blueprints/dataset/routes.py +++ b/app/blueprints/dataset/routes.py @@ -4,9 +4,11 @@ import shutil import tempfile import uuid -from datetime import datetime +from datetime import datetime, timezone from zipfile import ZipFile +from app import db + from flask import ( render_template, request, @@ -20,7 +22,15 @@ import app from app.blueprints.dataset.forms import DataSetForm -from app.blueprints.dataset.models import DataSet, PublicationType +from app.blueprints.dataset.models import ( + DSDownloadRecord, + DSViewRecord, + DataSet, + File, + FileDownloadRecord, + FileViewRecord, + PublicationType +) from app.blueprints.dataset import dataset_bp from app.blueprints.dataset.services import ( AuthorService, @@ -360,13 +370,21 @@ def download_dataset(dataset_id): mimetype="application/zip", ) - # Record the download in your database - DSDownloadRecordService().create( + # Check if the download record already exists for this cookie + existing_record = DSDownloadRecord.query.filter_by( user_id=current_user.id if current_user.is_authenticated else None, dataset_id=dataset_id, - download_date=datetime.now(datetime.UTC), - download_cookie=user_cookie, - ) + download_cookie=user_cookie + ).first() + + if not existing_record: + # Record the download in your database + DSDownloadRecordService().create( + user_id=current_user.id if current_user.is_authenticated else None, + dataset_id=dataset_id, + download_date=datetime.now(timezone.utc), + download_cookie=user_cookie, + ) return resp @@ -380,13 +398,21 @@ def view_dataset(dataset_id): if not user_cookie: user_cookie = str(uuid.uuid4()) - # Record the view in your database - DSViewRecordService().create( + # Check if the view record already exists for this cookie + existing_record = DSViewRecord.query.filter_by( user_id=current_user.id if current_user.is_authenticated else None, dataset_id=dataset_id, - view_date=datetime.now(datetime.UTC), - view_cookie=user_cookie, - ) + view_cookie=user_cookie + ).first() + + if not existing_record: + # Record the view in your database + DSViewRecordService().create( + user_id=current_user.id if current_user.is_authenticated else None, + dataset_id=dataset_id, + view_date=datetime.now(timezone.utc), + view_cookie=user_cookie, + ) # Save the cookie to the user's browser resp = make_response(render_template("dataset/view_dataset.html", dataset=dataset)) @@ -409,13 +435,21 @@ def download_file(file_id): if not user_cookie: user_cookie = str(uuid.uuid4()) - # Record the download in your database - FileDownloadRecordService().create( + # Check if the download record already exists for this cookie + existing_record = FileDownloadRecord.query.filter_by( user_id=current_user.id if current_user.is_authenticated else None, file_id=file_id, - download_date=datetime.now(datetime.UTC), - download_cookie=user_cookie, - ) + download_cookie=user_cookie + ).first() + + if not existing_record: + # Record the download in your database + FileDownloadRecordService().create( + user_id=current_user.id if current_user.is_authenticated else None, + file_id=file_id, + download_date=datetime.now(timezone.utc), + download_cookie=user_cookie, + ) # Save the cookie to the user's browser resp = make_response( @@ -426,6 +460,55 @@ def download_file(file_id): return resp +@dataset_bp.route('/file/view/', methods=['GET']) +def view_file(file_id): + file = File.query.get_or_404(file_id) + filename = file.name + + directory_path = f"uploads/user_{file.feature_model.data_set.user_id}/dataset_{file.feature_model.data_set_id}/" + parent_directory_path = os.path.dirname(current_app.root_path) + file_path = os.path.join(parent_directory_path, directory_path, filename) + + try: + if os.path.exists(file_path): + with open(file_path, 'r') as f: + content = f.read() + + user_cookie = request.cookies.get('view_cookie') + if not user_cookie: + user_cookie = str(uuid.uuid4()) + + # Check if the view record already exists for this cookie + existing_record = FileViewRecord.query.filter_by( + user_id=current_user.id if current_user.is_authenticated else None, + file_id=file_id, + view_cookie=user_cookie + ).first() + + if not existing_record: + # Register file view + new_view_record = FileViewRecord( + user_id=current_user.id if current_user.is_authenticated else None, + file_id=file_id, + view_date=datetime.now(), + view_cookie=user_cookie + ) + db.session.add(new_view_record) + db.session.commit() + + # Prepare response + response = jsonify({'success': True, 'content': content}) + if not request.cookies.get('view_cookie'): + response = make_response(response) + response.set_cookie('view_cookie', user_cookie, max_age=60*60*24*365*2) + + return response + else: + return jsonify({'success': False, 'error': 'File not found'}), 404 + except Exception as e: + return jsonify({'success': False, 'error': str(e)}), 500 + + @dataset_bp.route("/doi//", methods=["GET"]) def subdomain_index(doi): # Busca el dataset por DOI @@ -442,7 +525,7 @@ def subdomain_index(doi): DSViewRecordService().create( user_id=current_user.id if current_user.is_authenticated else None, dataset_id=dataset_id, - view_date=datetime.now(datetime.UTC), + view_date=datetime.now(timezone.utc), view_cookie=user_cookie, ) From 12996372c8b2692180f96f7cf196542fe794ee7f Mon Sep 17 00:00:00 2001 From: David Romero Date: Wed, 3 Jul 2024 18:58:01 +0200 Subject: [PATCH 7/7] fix: Fix linter --- rosemary/commands/coverage.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rosemary/commands/coverage.py b/rosemary/commands/coverage.py index 6aaaf82ce..6870e8516 100644 --- a/rosemary/commands/coverage.py +++ b/rosemary/commands/coverage.py @@ -19,7 +19,7 @@ def coverage(module_name, html): else: click.echo("Running coverage for all modules...") - coverage_cmd = ['pytest', '--ignore-glob=*selenium*','--cov=' + test_path, test_path] + coverage_cmd = ['pytest', '--ignore-glob=*selenium*', '--cov=' + test_path, test_path] if html: coverage_cmd.extend(['--cov-report', 'html'])