task/WG-295-WG-179-WG-185: fix and improve projects with duplicate sy…

…stem path constraints (#216)

* Update docker command

* Update project update route

* Refactor to remove ObservableDataProject

* Fix test

* Add restart nginx command

* Refactor the watch method into two methods

We can refresh users more often so setting that to every 30 minutes.

* Add script to check projects

* Rework namings of variables for iterating over users/project-users

* Rename variable

* Rename some variables

* Rename refresh methods

* Rename proj to project

* Fix rename

* Fix liting error

* Set Access-Control-Max-Age to 86400 as mas value for Firefox

Firefox had the longest max among browsers.

* Refactor project creation

* Add note about what project attributes can be updated

* Rework migrations so that they have date and name

* Improve checking script

Makefile

@@ -20,6 +20,11 @@ stop:
 restart-workers:  ## Restart workers
 	docker compose -f devops/docker-compose.local.yml --env-file .env restart workers
 
+.PHONY: restart-nginx
+restart-nginx:  ## Restart nginx
+	docker compose -f devops/docker-compose.local.yml --env-file .env restart nginx
+
+
 .PHONY: build
 build:
 	make geoapi && make workers

README.md

@@ -70,7 +70,8 @@ Then, create migrations:
 
 ```
 docker exec -it geoapi /bin/bash
-alembic revision --autogenerate
+# determine a description for the migration like 'add_user_email_column'
+alembic revision --autogenerate -m "add_user_email_column"
 # Then:
 # - remove drop table commands for postgis
 # - add/commit migrations

devops/geoapi-services/nginx.conf

@@ -38,15 +38,15 @@ http {
       large_client_header_buffers 2 50k;
 
       location / {
-           add_header "Access-Control-Allow-Origin"  *;
+          add_header 'Access-Control-Allow-Origin' '*' always;
 
           # Preflighted requests
-          if ($request_method = OPTIONS ) {
-              add_header "Access-Control-Allow-Origin"  *;
-              add_header "Access-Control-Allow-Methods" "GET, POST, OPTIONS, HEAD, PUT, DELETE";
-              add_header "Access-Control-Allow-Headers" "*";
-              add_header 'Access-Control-Max-Age' 1728000;
-              add_header 'Content-Length' 0;
+          if ($request_method = 'OPTIONS') {
+              add_header 'Access-Control-Allow-Origin' '*' always;
+              add_header 'Access-Control-Allow-Methods' 'GET, POST, OPTIONS, HEAD, PUT, DELETE' always;
+              add_header 'Access-Control-Allow-Headers' '*' always;
+              add_header 'Access-Control-Max-Age' 86400 always;
+              add_header 'Content-Length' 0 always;
               return 204;
           }
           rewrite ^/api(.*) /$1 break;
@@ -70,7 +70,7 @@ http {
               add_header "Access-Control-Allow-Origin"  *;
               add_header "Access-Control-Allow-Methods" "GET, POST, OPTIONS, HEAD, PUT, DELETE";
               add_header "Access-Control-Allow-Headers" "*";
-              add_header 'Access-Control-Max-Age' 1728000;
+              add_header 'Access-Control-Max-Age' 86400;
               add_header 'Content-Length' 0;
               return 204;
           }

devops/local_conf/nginx.conf

@@ -13,18 +13,20 @@ http {
     server {
         include /etc/nginx/mime.types;
         client_max_body_size 10g;
+
         location / {
-             add_header "Access-Control-Allow-Origin"  *;
+            add_header 'Access-Control-Allow-Origin' '*' always;
 
             # Preflighted requests
             if ($request_method = OPTIONS ) {
-                add_header "Access-Control-Allow-Origin"  *;
-                add_header "Access-Control-Allow-Methods" "GET, POST, OPTIONS, HEAD, PUT, DELETE";
-                add_header "Access-Control-Allow-Headers" "*";
-                add_header 'Access-Control-Max-Age' 1728000;
-                add_header 'Content-Length' 0;
+                add_header 'Access-Control-Allow-Origin' '*' always;
+                add_header 'Access-Control-Allow-Methods' 'GET, POST, OPTIONS, HEAD, PUT, DELETE' always;
+                add_header 'Access-Control-Allow-Headers' '*' always;
+                add_header 'Access-Control-Max-Age' 86400 always;
+                add_header 'Content-Length' 0 always;
                 return 204;
             }
+
             rewrite ^/api(.*) /$1 break;
             proxy_pass http://geoapi:8000;
             proxy_http_version 1.1;
@@ -39,13 +41,14 @@ http {
         location /assets {
             max_ranges 0;
             expires 30d;
-            add_header "Access-Control-Allow-Origin"  *;
+            add_header 'Access-Control-Allow-Origin' '*';
+
             # Preflighted requests
             if ($request_method = OPTIONS ) {
-                add_header "Access-Control-Allow-Origin"  *;
-                add_header "Access-Control-Allow-Methods" "GET, POST, OPTIONS, HEAD, PUT, DELETE";
-                add_header "Access-Control-Allow-Headers" "*";
-                add_header 'Access-Control-Max-Age' 1728000;
+                add_header 'Access-Control-Allow-Origin' '*';
+                add_header 'Access-Control-Allow-Methods' 'GET, POST, OPTIONS, HEAD, PUT, DELETE';
+                add_header 'Access-Control-Allow-Headers' '*';
+                add_header 'Access-Control-Max-Age' 86400;
                 add_header 'Content-Length' 0;
                 return 204;
             }

geoapi/alembic.ini

@@ -6,6 +6,7 @@ script_location = migrations
 
 # template used to generate migration files
 # file_template = %%(rev)s_%%(slug)s
+file_template = %%(year)d%%(month).2d%%(day).2d_%%(hour).2d%%(minute).2d-%%(rev)s_%%(slug)s
 
 # timezone to use when rendering the date
 # within the migration file as well as the filename.

geoapi/app.py

@@ -4,7 +4,7 @@
 from geoapi.settings import settings as app_settings
 from geoapi.db import db_session
 from geoapi.exceptions import (InvalidGeoJSON, InvalidEXIFData, InvalidCoordinateReferenceSystem,
-                               ObservableProjectAlreadyExists, ApiException, StreetviewAuthException,
+                               ProjectSystemPathWatchFilesAlreadyExists, ApiException, StreetviewAuthException,
                                StreetviewLimitException, AuthenticationIssue)
 
 import logging
@@ -48,9 +48,9 @@ def handle_coordinate_reference_system_exception(error: Exception):
     return {'message': 'Invalid data, coordinate reference system could not be found'}, 400
 
 
-@api.errorhandler(ObservableProjectAlreadyExists)
-def handle_observable_project_already_exists_exception(error: Exception):
-    return {'message': 'Conflict, a project for this storage system/path already exists'}, 409
+@api.errorhandler(ProjectSystemPathWatchFilesAlreadyExists)
+def handle_project_system_path_watch_files_already_exists_exception(error: Exception):
+    return {'message': 'Conflict, a project watching files for this storage system/path already exists'}, 409
 
 
 @api.errorhandler(StreetviewAuthException)

geoapi/celery_app.py

@@ -1,7 +1,8 @@
 from celery import Celery
-from celery.schedules import crontab
+from datetime import timedelta
 from geoapi.settings import settings
 
+
 CELERY_CONNECTION_STRING = "amqp://{user}:{pwd}@{hostname}/{vhost}".format(
     user=settings.RABBITMQ_USERNAME,
     pwd=settings.RABBITMQ_PASSWD,
@@ -15,8 +16,12 @@
              include=['geoapi.tasks'])
 
 app.conf.beat_schedule = {
-    'refresh_observable_projects': {
-        'task': 'geoapi.tasks.external_data.refresh_observable_projects',
-        'schedule': crontab(hour='*', minute='0')
+    'refresh_projects_watch_content': {
+        'task': 'geoapi.tasks.external_data.refresh_projects_watch_content',
+        'schedule': timedelta(hours=1)
+    },
+    'refresh_projects_watch_users': {
+        'task': 'geoapi.tasks.external_data.refresh_projects_watch_users',
+        'schedule': timedelta(minutes=30)
     }
 }

geoapi/exceptions.py

@@ -14,8 +14,8 @@ class InvalidCoordinateReferenceSystem(Exception):
     pass
 
 
-class ObservableProjectAlreadyExists(Exception):
-    """ Observable Project already exists for this path"""
+class ProjectSystemPathWatchFilesAlreadyExists(Exception):
+    """ Project with watch_files True already exists for this system path"""
     pass
 
 

geoapi/migrations/versions/20240916_1855-968f358e102a_add_watch_variables_to_project.py

@@ -0,0 +1,68 @@
+"""add_watch_variables_to_project
+
+Revision ID: 968f358e102a
+Revises: 4eeeeea72dbc
+Create Date: 2024-09-16 18:55:13.685590
+
+"""
+from alembic import op
+import sqlalchemy as sa
+from sqlalchemy.orm import Session
+from geoapi.log import logger
+
+
+# revision identifiers, used by Alembic.
+revision = '968f358e102a'
+down_revision = '4eeeeea72dbc'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # Add new columns
+    op.add_column('projects', sa.Column('watch_content', sa.Boolean(), nullable=True))
+    op.add_column('projects', sa.Column('watch_users', sa.Boolean(), nullable=True))
+
+    bind = op.get_bind()
+    session = Session(bind=bind)
+
+    try:
+        # Query all projects and their related observable data projects in order
+        # to set the watch_content and watch_users
+        projects_query = sa.text("""
+            SELECT p.id, odp.id as odp_id, odp.watch_content
+            FROM projects p
+            LEFT JOIN observable_data_projects odp ON p.id = odp.project_id
+        """)
+        results = session.execute(projects_query)
+
+        # Update projects based on the query results
+        for project_id, odp_id, odp_watch_content in results:
+            update_query = sa.text("""
+                UPDATE projects
+                SET watch_content = :watch_content, watch_users = :watch_users
+                WHERE id = :project_id
+            """)
+            session.execute(update_query, {
+                'watch_content': odp_watch_content if odp_id is not None else False,
+                'watch_users': odp_id is not None,
+                'project_id': project_id
+            })
+        session.commit()
+        logger.info(f"Data migration of project/observable completed successfully")
+
+    except Exception as e:
+        session.rollback()
+        logger.exception(f"An error occurred during project/observable data migration: {str(e)}")
+        raise
+    finally:
+        session.close()
+
+    # Drop the unique constraint
+    op.drop_constraint('observable_data_projects_system_id_path_key', 'observable_data_projects')
+
+
+def downgrade():
+    op.drop_column('projects', 'watch_users')
+    op.drop_column('projects', 'watch_content')
+    op.create_unique_constraint('observable_data_projects_system_id_path_key', 'observable_data_projects', ['system_id', 'path'])

geoapi/models/observable_data.py

@@ -1,17 +1,15 @@
 from sqlalchemy import (
     Column, Integer, String, Boolean,
-    ForeignKey, DateTime, UniqueConstraint
+    ForeignKey, DateTime
 )
 from sqlalchemy.orm import relationship
 from sqlalchemy.sql import func
 from geoapi.db import Base
 
 
+# Deprecated; Replaced by watch_user and watch_content of Project table. See https://tacc-main.atlassian.net/browse/WG-377
 class ObservableDataProject(Base):
     __tablename__ = 'observable_data_projects'
-    __table_args__ = (
-        UniqueConstraint('system_id', 'path', name="unique_system_id_path"),
-    )
     id = Column(Integer, primary_key=True)
     project_id = Column(ForeignKey('projects.id', ondelete="CASCADE", onupdate="CASCADE"), index=True)
     created_date = Column(DateTime(timezone=True), server_default=func.now())

geoapi/models/project.py

@@ -29,10 +29,11 @@ class Project(Base):
     id = Column(Integer, primary_key=True)
     uuid = Column(UUID(as_uuid=True), default=uuid.uuid4, nullable=False)
     tenant_id = Column(String, nullable=False)
-    # Project system_id/system_path really not used except for analytics.
-    # This could be improved; see https://jira.tacc.utexas.edu/browse/WG-185
+    # associated tapis system id
     system_id = Column(String, nullable=True)
+    # associated tapis system path
     system_path = Column(String, nullable=True)
+    # associated tapis system file
     system_file = Column(String, nullable=True)
     name = Column(String, nullable=False)
     description = Column(String)
@@ -47,5 +48,16 @@ class Project(Base):
                          overlaps="project,project_users")
     point_clouds = relationship('PointCloud', cascade="all, delete-orphan")
 
+    # watch content of tapis directory location (system_id and system_path)
+    watch_content = Column(Boolean, default=False)
+
+    # watch user of tapis system (system_id)
+    watch_users = Column(Boolean, default=False)
+
     def __repr__(self):
-        return '<Project(id={})>'.format(self.id)
+        return f"<Project(id={self.id}, uuid={self.uuid}, tenant_id='{self.tenant_id}', " \
+               f"system_id='{self.system_id}', system_path='{self.system_path}', " \
+               f"system_file='{self.system_file}', name='{self.name}', " \
+               f"description='{self.description}', public={self.public}, " \
+               f"created={self.created}, updated={self.updated}, " \
+               f"watch_content={self.watch_content}, watch_users={self.watch_users})>"

geoapi/routes/projects.py

@@ -60,6 +60,12 @@
     'watch_users': fields.Boolean()
 })
 
+project_update_payload = api.model('Project', {
+    'name': fields.String(),
+    'description': fields.String(),
+    'public': fields.Boolean(),
+})
+
 project_response = api.model('ProjectResponse', {
     'id': fields.Integer(),
     'uuid': fields.String(),
@@ -225,14 +231,14 @@ def delete(self, projectId: int):
 
     @api.doc(id="updateProject",
              description="Update metadata about a project")
+    @api.expect(project_update_payload)
     @api.marshal_with(project_response)
     @project_permissions
     def put(self, projectId: int):
         u = request.current_user
         logger.info("Update project:{} for user:{}".format(projectId,
                                                            u.username))
         return ProjectsService.update(db_session,
-                                      user=u,
                                       projectId=projectId,
                                       data=api.payload)