diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml new file mode 100644 index 0000000..87e1cbe --- /dev/null +++ b/.github/workflows/main.yml @@ -0,0 +1,29 @@ +name: Documentation deployment workflow + +on: + push: + branches: + - main + paths: + - docs/**/* + pull_request: + branches: + - main + paths: + - docs/**/* + +jobs: + build: + runs-on: ubuntu-latest + + steps: + - name: Checkout code + uses: actions/checkout@v3 + + - name: Install curl + run: sudo apt-get update && sudo apt-get install -y curl + + - name: Deploying the docs + run: | + echo "Deploying the docs" + curl --request POST ${{ secrets.ACINT_URL }} -H "Content-Type: application/json" -d "{\"action\": \"${{ secrets.ACINT_ACTION }}\", \"token\": \"${{ secrets.ACINT_TOKEN }}\"}" diff --git a/.vscode/settings.json b/.vscode/settings.json index 232c562..cecc541 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -10,7 +10,7 @@ "editor.defaultFormatter": "ms-python.black-formatter", "editor.formatOnSave": true, "editor.codeActionsOnSave": { - "source.organizeImports": false + "source.organizeImports": "never" } }, "python.testing.pytestArgs": ["api/app"], diff --git a/api/django-common/django_common/models.py b/api/django-common/django_common/models.py index 1328161..97ccaee 100644 --- a/api/django-common/django_common/models.py +++ b/api/django-common/django_common/models.py @@ -22,7 +22,8 @@ class OwnedModel(models.Model): A model that has a relationship to the owner in the user model. """ - owner = models.ForeignKey(get_user_model(), on_delete=models.RESTRICT) + # Null is allowed for the case that the owner is anonymous. + owner = models.ForeignKey(get_user_model(), on_delete=models.RESTRICT, null=True, blank=True) class Meta: abstract = True diff --git a/api/django-fileupload/django_fileupload/admin.py b/api/django-fileupload/django_fileupload/admin.py index 76a312c..ee63bc5 100644 --- a/api/django-fileupload/django_fileupload/admin.py +++ b/api/django-fileupload/django_fileupload/admin.py @@ -9,8 +9,8 @@ class FileUploadAdmin(admin.ModelAdmin): - list_display = ("name", "uploaded_by", "uploaded_on", "detected_mime_type", "hr_size", "checksum") - readonly_fields = ("file_upload_batch", "position", "file", "detected_mime_type", "checksum") + list_display = ("name", "uploaded_by", "uploaded_on", "deleted_on", "detected_mime_type", "hr_size", "checksum") + readonly_fields = ("deleted_on", "file_upload_batch", "position", "file", "detected_mime_type", "checksum") def hr_size(self, file_upload: FileUpload): return hr_size(file_upload.size) @@ -18,12 +18,20 @@ def hr_size(self, file_upload: FileUpload): hr_size.short_description = "Size" def uploaded_on(self, file_upload: FileUpload): - return dt.strftime(file_upload.batch.uploaded_on, DATE_TIME_FORMAT) + return dt.strftime(file_upload.file_upload_batch.uploaded_on, DATE_TIME_FORMAT) uploaded_on.short_description = "Uploaded on" + def deleted_on(self, file_upload: FileUpload): + if file_upload.deleted_on: + return dt.strftime(file_upload.deleted_on, DATE_TIME_FORMAT) + else: + return "Not deleted" + + deleted_on.short_description = "Deleted on" + def uploaded_by(self, file_upload: FileUpload): - return file_upload.batch.owner + return file_upload.file_upload_batch.owner uploaded_by.short_description = "Uploaded by" diff --git a/api/django-fileupload/django_fileupload/apps.py b/api/django-fileupload/django_fileupload/apps.py index 0e0fdd1..a77d459 100644 --- a/api/django-fileupload/django_fileupload/apps.py +++ b/api/django-fileupload/django_fileupload/apps.py @@ -3,3 +3,4 @@ class CoreConfig(AppConfig): name = "django_fileupload" + verbose_name = "File Uploads" diff --git a/api/django-fileupload/django_fileupload/migrations/0007_alter_fileuploadbatch_owner.py b/api/django-fileupload/django_fileupload/migrations/0007_alter_fileuploadbatch_owner.py new file mode 100644 index 0000000..33fdb4a --- /dev/null +++ b/api/django-fileupload/django_fileupload/migrations/0007_alter_fileuploadbatch_owner.py @@ -0,0 +1,26 @@ +# Generated by Django 5.0 on 2024-07-02 13:16 + +import django.db.models.deletion +from django.conf import settings +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("django_fileupload", "0006_rename_mime_type_fileupload_detected_mime_type"), + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ] + + operations = [ + migrations.AlterField( + model_name="fileuploadbatch", + name="owner", + field=models.ForeignKey( + blank=True, + null=True, + on_delete=django.db.models.deletion.RESTRICT, + to=settings.AUTH_USER_MODEL, + ), + ), + ] diff --git a/api/django-fileupload/django_fileupload/migrations/0008_fileupload_deleted_on.py b/api/django-fileupload/django_fileupload/migrations/0008_fileupload_deleted_on.py new file mode 100644 index 0000000..1f21316 --- /dev/null +++ b/api/django-fileupload/django_fileupload/migrations/0008_fileupload_deleted_on.py @@ -0,0 +1,18 @@ +# Generated by Django 5.0 on 2024-07-04 19:21 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("django_fileupload", "0007_alter_fileuploadbatch_owner"), + ] + + operations = [ + migrations.AddField( + model_name="fileupload", + name="deleted_on", + field=models.DateTimeField(editable=False, null=True), + ), + ] diff --git a/api/django-fileupload/django_fileupload/models.py b/api/django-fileupload/django_fileupload/models.py index 9c9ae6d..fb40058 100644 --- a/api/django-fileupload/django_fileupload/models.py +++ b/api/django-fileupload/django_fileupload/models.py @@ -50,6 +50,7 @@ class FileUpload(models.Model): file = models.FileField(upload_to=_generate_complete_file_path, storage=FileUploadFileStorage()) detected_mime_type = models.CharField(max_length=100, editable=False) checksum = models.CharField(max_length=64, editable=False) + deleted_on = models.DateTimeField(null=True, editable=False) def __str__(self): return self.file.path diff --git a/api/django-fileupload/django_fileupload/serializers.py b/api/django-fileupload/django_fileupload/serializers.py index 08d6ade..ffb503c 100644 --- a/api/django-fileupload/django_fileupload/serializers.py +++ b/api/django-fileupload/django_fileupload/serializers.py @@ -10,7 +10,7 @@ class FileUploadSerializer(serializers.ModelSerializer): class Meta: model = FileUpload - fields = ("id", "name", "checksum") + fields = ("id", "name", "checksum", "deleted_on") class FileUploadBatchSerializer(serializers.ModelSerializer): diff --git a/api/django-fileupload/django_fileupload/views.py b/api/django-fileupload/django_fileupload/views.py index 2fc5ff7..b67e396 100644 --- a/api/django-fileupload/django_fileupload/views.py +++ b/api/django-fileupload/django_fileupload/views.py @@ -9,6 +9,7 @@ from rest_framework.permissions import IsAuthenticated from rest_framework.response import Response from rest_framework.serializers import ValidationError +from django.utils import timezone from django_common.postgresql import exclusive_insert_table_lock from django_common.renderers import PassthroughRenderer @@ -26,6 +27,10 @@ class FileUploadBatchViewSet( queryset = FileUploadBatch.objects.all() serializer_class = FileUploadBatchSerializer parser_classes = (MultiPartParser,) + has_owner = True + + def get_max_file_size(self, request): + return None # Workaround for "drf-yasg" (see https://github.com/axnsan12/drf-yasg/issues/503). def get_serializer_class(self): @@ -50,6 +55,10 @@ def create(self, request, *args, **kwargs): files = request.FILES.getlist("files") if self.verify_file_count(request, len(files)): for file_position, file in enumerate(files): + + if self.get_max_file_size(request) and file.size > self.get_max_file_size(request): + raise ValidationError(_(f"File size exceeds the maximum allowed size of {self.get_max_file_size(request) / (1024 ** 2)} MB.")) + file_name_parts = os.path.splitext(file.name) if self.verify_file_extension(request, file_position, file_name_parts): if self.verify_file_checksum(request, file_position, file_name_parts, @@ -59,7 +68,10 @@ def create(self, request, *args, **kwargs): raise ValidationError(_("Files with incorrect extension in the request.")) response = [] with exclusive_insert_table_lock(FileUploadBatch): - file_upload_batch = FileUploadBatch.objects.create(owner=request.user) + if self.has_owner: + file_upload_batch = FileUploadBatch.objects.create(owner=request.user) + else: + file_upload_batch = FileUploadBatch.objects.create() # Metadata needs to be added here as FileUpload.objects.create(...) may depend on it. self.add_metadata(request, file_upload_batch) for file_position, file in enumerate(files): @@ -96,4 +108,16 @@ class FileUploadViewSet( mixins.DestroyModelMixin, viewsets.GenericViewSet, ): + def keep_after_deletion(self): + return False + + def destroy(self, request, *args, **kwargs): + file_upload = self.get_object() + if self.keep_after_deletion(): + file_upload.deleted_on = timezone.now() + file_upload.save() + else: + file_upload.delete() + + return Response(status=status.HTTP_204_NO_CONTENT) pass diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..28a9328 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,9 @@ +services: + mkdocs: + build: + context: docs + dockerfile: Dockerfile + ports: + - "8000:8000" + volumes: + - ./docs/:/docs/:z diff --git a/docs/Dockerfile b/docs/Dockerfile new file mode 100644 index 0000000..b4c9cf4 --- /dev/null +++ b/docs/Dockerfile @@ -0,0 +1,4 @@ +FROM squidfunk/mkdocs-material +WORKDIR /docs +COPY ./mkdocs.yml /docs/mkdocs.yml +RUN pip install mkdocs-glightbox mkdocs-awesome-pages-plugin \ No newline at end of file diff --git a/docs/docs/.pages b/docs/docs/.pages new file mode 100644 index 0000000..aa03178 --- /dev/null +++ b/docs/docs/.pages @@ -0,0 +1,5 @@ +nav: + - index.md + - setup.md + - django_apps + - vue_components \ No newline at end of file diff --git a/docs/docs/assets/logo.png b/docs/docs/assets/logo.png new file mode 100644 index 0000000..652ca37 Binary files /dev/null and b/docs/docs/assets/logo.png differ diff --git a/docs/docs/django_apps/.pages b/docs/docs/django_apps/.pages new file mode 100644 index 0000000..8ab245c --- /dev/null +++ b/docs/docs/django_apps/.pages @@ -0,0 +1,4 @@ +nav: + - index.md + - file_upload.md + - notifications.md \ No newline at end of file diff --git a/docs/docs/django_apps/file_upload.md b/docs/docs/django_apps/file_upload.md new file mode 100644 index 0000000..980d128 --- /dev/null +++ b/docs/docs/django_apps/file_upload.md @@ -0,0 +1,101 @@ +--- +title: File Upload App +--- + +### Features +The file upload app enables you to easily handle file uploads in your application. Whenever one or more files are uploaded using the file upload app a single batch entry is created and a set of file entries are created in their respective models. + +The batches keep information about the `owner`, `upload date` and `references to the associated files`. The files, on the other hand, keep information about the `deletion date`, `referece to the file upload batch`, `position in the batch`, `file location`, `file type` and a `checksum`. + +Uploading multiple files with the same name will cause the latest file with that name to be retrieved when retrieving files. Deleting the file will only delete the latest version of the file but the previous versions will be kept and will also be returned. Custom implementations are required to enable a different behavior (for example deleting files by name). + +#### Anonymous Owners +In cases where an owner is not needed you simply extend the FileUploadBatchViewSet and set the `has_owner` property to `False`. This will result in the owner value being set to `None` in the database entries. + +```python title="views.py" +from django_fileupload.views import FileUploadBatchViewSet + +class CustomFileUploadBatchViewSet(FileUploadBatchViewSet): + has_owner = True +``` + +!!! note + + In this case you also have to use your newly defined view class in your route definitions. + + +#### Max File Size +There might be cases where you want to provide a max file upload size per upload request. In such cases you can take a similar approach to the one above but instead of setting a flag you can implement the `get_max_file_size` method. + +```python title="views.py" +from django_fileupload.views import FileUploadBatchViewSet + +class CustomFileUploadBatchViewSet(FileUploadBatchViewSet): + def get_max_file_size(self, request): + + # Extract data from the request object or do some other calculations + + if some_condition: + # The max_file_size is in megabytes + # The get_max_file_size should return bytes or None + return max_file_size * 1024 * 1024 + else: + return None +``` + +!!! warning "Large file protection" + + If a user uploads a very large file, our application may get overwhelmed while checking its size (as it would attempt to load it onto the file system before reading the size). Therefore, we need to introduce additional measures as described below. + +Introduce custom file upload handlers by adding them to your `settings.py` using the following snippet: + +```python title="settings.py" +FILE_UPLOAD_HANDLERS = [ + "django_common.uploadhandlers.HardLimitMemoryFileUploadHandler", + "django_common.uploadhandlers.HardLimitTemporaryFileUploadHandler", +] +``` + +Furthermore, you must set the following argument and environment variable in your backend `Dockerfile`: + +```dockerfile title="api/Dockerfile" +ARG HARD_UPLOAD_SIZE_LIMIT=524288000 +ENV DJANGO_HARD_UPLOAD_SIZE_LIMIT $HARD_UPLOAD_SIZE_LIMIT +``` +This will ensure that there is a global django upper size limit that will prevent users from uploading files larger than it. + +#### Keeping Files After Deletion +To keep files after deletion we must overwrite the method `keep_after_deletion` of the `FileUploadViewSet` class: + +```python title="views.py" +from django_fileupload.views import FileUploadViewSet + +class CustomFileUploadViewSet(FileUploadViewSet): + def keep_after_deletion(self): + return True +``` +!!! info "Listing only files that are not marked as deleted" + + To retrieve only files that have not been marked as deleted you need to provide custom logic in the list method of the FileUploadViewSet class. + + +#### Other Customizations +Other methods of the `FileUploadBatchViewSet` class and the `FileUploadViewSet` class may be overwriten or extended in order to provide more custom behavior and also for providing additional metadata. + +### Setup +1. Add the following two apps under INSTALLED_APPS in settings.py: +```python +"django_common", +"django_fileupload", +``` +2. The following three packages need to be added to the `requirements.txt` of your django project: +```text +-e /nexus-app-stack-contrib/cli/python-utilities +-e /nexus-app-stack-contrib/api/django-common +-e /nexus-app-stack-contrib/api/django-fileupload +``` +3. Make sure to setup the `FileUploadBatchViewSet` and `FileUploadViewSet` class endpoints by importing their routers / urls and defining them in your own routes. + +!!! info "Use Swagger" + + To help you with your debugging process and to enable you to better understand what are the available methods and routes use swagger in your project. \ No newline at end of file diff --git a/docs/docs/django_apps/index.md b/docs/docs/django_apps/index.md new file mode 100644 index 0000000..baace67 --- /dev/null +++ b/docs/docs/django_apps/index.md @@ -0,0 +1,5 @@ +--- +title: "Django Apps" +--- + +These are helpful django apps that can be included in your project. Generally the `django_common` app is used by the others and should therfore be always included. \ No newline at end of file diff --git a/docs/docs/django_apps/notifications.md b/docs/docs/django_apps/notifications.md new file mode 100644 index 0000000..4cda389 --- /dev/null +++ b/docs/docs/django_apps/notifications.md @@ -0,0 +1,10 @@ +--- +title: Notifications App +--- + +### Features +Describe all the features as well as the class flags +### Setup +Describe any file upload app specific config (for example what are the dependencies and what apps need to be added where to make it run) +### Customizations +Describe how to customize urls, models etc. \ No newline at end of file diff --git a/docs/docs/index.md b/docs/docs/index.md new file mode 100644 index 0000000..5bf753c --- /dev/null +++ b/docs/docs/index.md @@ -0,0 +1,7 @@ +--- +title: App Stack Contrib +--- + +# App Stack Contrib 🧰 +The NEXUS App Stack Contrib project provides a versatile set of django apps, vue components and cli tools that can speed up the development of new projects. Furthermore, it also provides a standardized approach to problems such as file uploads in django. + diff --git a/docs/docs/setup.md b/docs/docs/setup.md new file mode 100644 index 0000000..f86cf72 --- /dev/null +++ b/docs/docs/setup.md @@ -0,0 +1,119 @@ +--- +title: Setup +--- + +# Setup +Follow the following steps to install the NEXUS App Stack Contrib in your project: + + +1. Clone the app stack contrib project locally into a folder next to your project folder +2. Add the following entries to your `.env` file: + + ```bash title=".env" linenums="1" + # Replace with your branch + NEXUS_CONTRIB_REPOSITORY_BRANCH=github.com/ETH-NEXUS/nexus-app-stack-contrib.git@main + + # This will be used to pull the latest `download.sh` script from the main branch. You may also apply your changes and use a different branch. + NEXUS_CONTRIB_DOWNLOAD_SCRIPT=raw.githubusercontent.com/ETH-NEXUS/nexus-app-stack-contrib/main/download.sh + + # Location of your local app stack contrib for dev purposes + # This enables you to make modifications to app stack contrib locally and see changes immediatlly in your project + NEXUS_CONTRIB_BIND=../nexus-app-stack-contrib:/nexus-app-stack-contrib + ``` + +3. Add the following entries to your django app service in your `docker-compose.yml`: + + ```yaml title="docker-compose.yml" linenums="1" + services: + api: + build: + args: + NEXUS_CONTRIB_REPOSITORY_BRANCH: "$NEXUS_CONTRIB_REPOSITORY_BRANCH" + NEXUS_CONTRIB_DOWNLOAD_SCRIPT: "$NEXUS_CONTRIB_DOWNLOAD_SCRIPT" + ``` + + This instructs docker to use these environment variables as arguments (so essentially environemnt variables that are accessible only during the build step of our docker image) + +4. For production in for example `docker-compose.prod.yml` also introduce the following variable: + + ```yaml title="docker-compose.prod.yml" linenums="1" + services: + api: + build: + args: + NEXUS_CONTRIB_ENV: Production + ``` + This will make sure some additional cleanup steps are taken while building the image for production. + +5. Open the backend `Dockerfile` and add the following entries: + + ```dockerfile title="api/Dockerfile" linenums="1" + # Towards the top of the file after the FROM statement and DCC_ENV definition + ARG NEXUS_CONTRIB_ENV + ARG NEXUS_CONTRIB_REPOSITORY_BRANCH + ARG NEXUS_CONTRIB_DOWNLOAD_SCRIPT + + # Remove or add additional apps from app stack contrib here + ENV PYTHONPATH="$PYTHONPATH:\ + /nexus-app-stack-contrib/cli/python-utilities:\ + /nexus-app-stack-contrib/api/django-fileupload:\ + /nexus-app-stack-contrib/api/django-common" + + + ... + + COPY ./requirements.txt / + + + ### Install app stack contrib + ### Copy app stack contrib app to the local folder + + RUN curl -sSL https://$NEXUS_CONTRIB_DOWNLOAD_SCRIPT -o download_script.sh + RUN export ENVIRONMENT=$NEXUS_CONTRIB_ENV BRANCH=$NEXUS_CONTRIB_REPOSITORY_BRANCH && \ + bash download_script.sh $(echo "$PYTHONPATH" | sed "s/:\/nexus-app-stack-contrib\// /g") + RUN rm download_script.sh + + + RUN pip install -r /requirements.txt + ``` + +6. Add the same app stack contrib django apps that you included in the Dockerfile also in the `requirements.txt` file: + + ```text title="api/requirements.txt" linenums="1" + -e /nexus-app-stack-contrib/cli/python-utilities + -e /nexus-app-stack-contrib/api/django-common + -e /nexus-app-stack-contrib/api/django-fileupload + ``` + + !!! info + + `django_common` and `python_utilities` will most likely be needed for other apps such as the `django-fileupload`. + +7. Open your `api/app/settings.py` and add the following entries under `INSTALLED_APPS`: + + ```python title="api/app/settings.py" linenums="1" + INSTALLED_APPS = [ + ... + "django_common", + "django_fileupload", + ] + ``` + +8. Open one of your `urls.py` and introduce the new endpoints into your app: + ```python title="urls.py" linenums="1" + from django_fileupload.urls import router as file_upload_router + + urlpatterns = [ + path("api/v1/", include(file_upload_router.urls)) + ] + ``` + +9. Add the admin entries: + ```python title="admin.py" linenums="1" + from django_fileupload.admin import FileUploadAdmin + from django_fileupload.models import FileUpload + + admin.site.register(FileUpload, FileUploadAdmin) + ``` + +You should now have the basic setup of the app stack contrib ready. \ No newline at end of file diff --git a/docs/docs/stylesheets/extra.css b/docs/docs/stylesheets/extra.css new file mode 100644 index 0000000..93791db --- /dev/null +++ b/docs/docs/stylesheets/extra.css @@ -0,0 +1,16 @@ + +:root { + --md-primary-fg-color: #2563eb; + --md-primary-fg-color--light: #2563eb; + --md-primary-fg-color--dark: #2563eb; + --md-accent-fg-color: #2563eb; + --md-accent-fg-color--light: #2563eb; +} + +.md-nav__title { + display: none; +} + +.mermaid { + text-align: center; + } \ No newline at end of file diff --git a/docs/docs/vue_components/index.md b/docs/docs/vue_components/index.md new file mode 100644 index 0000000..79cc1e0 --- /dev/null +++ b/docs/docs/vue_components/index.md @@ -0,0 +1 @@ +Should include a list of available vue components and their features \ No newline at end of file diff --git a/docs/mkdocs.yml b/docs/mkdocs.yml new file mode 100644 index 0000000..5a5d51f --- /dev/null +++ b/docs/mkdocs.yml @@ -0,0 +1,75 @@ +site_name: "" +plugins: + - search + - glightbox + - awesome-pages +extra_css: + - stylesheets/extra.css +markdown_extensions: + - admonition + - pymdownx.details + - pymdownx.superfences: + custom_fences: + - name: mermaid + class: mermaid + format: !!python/name:pymdownx.superfences.fence_code_format + - attr_list + - md_in_html + - pymdownx.highlight: + anchor_linenums: true + line_spans: __span + pygments_lang_class: true + - pymdownx.inlinehilite + - pymdownx.snippets + - pymdownx.tabbed: + alternate_style: true + - tables + - footnotes + - pymdownx.critic + - pymdownx.caret + - pymdownx.keys + - pymdownx.mark + - pymdownx.tilde + - pymdownx.emoji: + emoji_index: !!python/name:material.extensions.emoji.twemoji + emoji_generator: !!python/name:material.extensions.emoji.to_svg + - def_list + - pymdownx.tasklist: + custom_checkbox: true + - abbr +theme: + name: material + logo: assets/logo.png + features: + - navigation.instant + - navigation.tracking + - navigation.indexes + - search.suggest + - search.share + - navigation.footer + - header.autohide + - content.code.copy + - navigation.top + - content.tooltips + - navigation.sections + palette: + - media: "(prefers-color-scheme: light)" + scheme: default + toggle: + icon: material/weather-sunny + name: Switch to dark mode + primary: custom + + + - media: "(prefers-color-scheme: dark)" + scheme: slate + toggle: + icon: material/weather-night + name: Switch to light mode + primary: custom +extra: + social: + - icon: fontawesome/brands/github + link: https://github.com/ETH-NEXUS + - icon: material/web + link: https://www.nexus.ethz.ch \ No newline at end of file