Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

task/WG-295-WG-179-WG-185: fix and improve projects with duplicate system path constraints #216

Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
22 commits
Select commit Hold shift + click to select a range
acfc223
Update docker command
nathanfranklin Aug 29, 2024
c4d6f2e
Update project update route
nathanfranklin Aug 29, 2024
aa6ca26
Refactor to remove ObservableDataProject
nathanfranklin Sep 5, 2024
86492f8
Fix test
nathanfranklin Sep 6, 2024
87e365d
Add restart nginx command
nathanfranklin Sep 6, 2024
06d6202
Refactor the watch method into two methods
nathanfranklin Sep 6, 2024
21ae56c
Merge branch 'main' into task/WG-295-WG-179-WG-185-fix-and-improve-pr…
nathanfranklin Sep 6, 2024
32d08b6
Add script to check projects
nathanfranklin Sep 6, 2024
ca484df
Rework namings of variables for iterating over users/project-users
nathanfranklin Sep 16, 2024
8af0d2f
Rename variable
nathanfranklin Sep 16, 2024
f2a4f84
Rename some variables
nathanfranklin Sep 16, 2024
e6abc45
Rename refresh methods
nathanfranklin Sep 16, 2024
6271d15
Rename proj to project
nathanfranklin Sep 16, 2024
8c94fae
Fix rename
nathanfranklin Sep 16, 2024
b81add5
Fix liting error
nathanfranklin Sep 16, 2024
e4bfad2
Set Access-Control-Max-Age to 86400 as mas value for Firefox
nathanfranklin Sep 16, 2024
2d740bc
Refactor project creation
nathanfranklin Sep 16, 2024
c62d493
Add note about what project attributes can be updated
nathanfranklin Sep 16, 2024
a75f4d8
Rework migrations so that they have date and name
nathanfranklin Sep 16, 2024
febc2b0
Improve checking script
nathanfranklin Sep 16, 2024
b145e9f
Merge branch 'main' into task/WG-295-WG-179-WG-185-fix-and-improve-pr…
nathanfranklin Sep 16, 2024
9510622
Merge branch 'main' into task/WG-295-WG-179-WG-185-fix-and-improve-pr…
nathanfranklin Sep 16, 2024
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,11 @@ stop:
restart-workers: ## Restart workers
docker compose -f devops/docker-compose.local.yml --env-file .env restart workers

.PHONY: restart-nginx
nathanfranklin marked this conversation as resolved.
Show resolved Hide resolved
restart-nginx: ## Restart nginx
docker compose -f devops/docker-compose.local.yml --env-file .env restart nginx


.PHONY: build
build:
make geoapi && make workers
Expand Down
3 changes: 2 additions & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -70,7 +70,8 @@ Then, create migrations:

```
docker exec -it geoapi /bin/bash
alembic revision --autogenerate
# determine a description for the migration like 'add_user_email_column'
alembic revision --autogenerate -m "add_user_email_column"
# Then:
# - remove drop table commands for postgis
# - add/commit migrations
Expand Down
16 changes: 8 additions & 8 deletions devops/geoapi-services/nginx.conf
Original file line number Diff line number Diff line change
Expand Up @@ -38,15 +38,15 @@ http {
large_client_header_buffers 2 50k;

location / {
add_header "Access-Control-Allow-Origin" *;
add_header 'Access-Control-Allow-Origin' '*' always;

# Preflighted requests
if ($request_method = OPTIONS ) {
add_header "Access-Control-Allow-Origin" *;
add_header "Access-Control-Allow-Methods" "GET, POST, OPTIONS, HEAD, PUT, DELETE";
add_header "Access-Control-Allow-Headers" "*";
add_header 'Access-Control-Max-Age' 1728000;
add_header 'Content-Length' 0;
if ($request_method = 'OPTIONS') {
add_header 'Access-Control-Allow-Origin' '*' always;
add_header 'Access-Control-Allow-Methods' 'GET, POST, OPTIONS, HEAD, PUT, DELETE' always;
add_header 'Access-Control-Allow-Headers' '*' always;
add_header 'Access-Control-Max-Age' 86400 always;
add_header 'Content-Length' 0 always;
return 204;
}
rewrite ^/api(.*) /$1 break;
Expand All @@ -70,7 +70,7 @@ http {
add_header "Access-Control-Allow-Origin" *;
add_header "Access-Control-Allow-Methods" "GET, POST, OPTIONS, HEAD, PUT, DELETE";
add_header "Access-Control-Allow-Headers" "*";
add_header 'Access-Control-Max-Age' 1728000;
add_header 'Access-Control-Max-Age' 86400;
add_header 'Content-Length' 0;
return 204;
}
Expand Down
25 changes: 14 additions & 11 deletions devops/local_conf/nginx.conf
Original file line number Diff line number Diff line change
Expand Up @@ -13,18 +13,20 @@ http {
server {
include /etc/nginx/mime.types;
client_max_body_size 10g;

location / {
add_header "Access-Control-Allow-Origin" *;
add_header 'Access-Control-Allow-Origin' '*' always;

# Preflighted requests
if ($request_method = OPTIONS ) {
add_header "Access-Control-Allow-Origin" *;
add_header "Access-Control-Allow-Methods" "GET, POST, OPTIONS, HEAD, PUT, DELETE";
add_header "Access-Control-Allow-Headers" "*";
add_header 'Access-Control-Max-Age' 1728000;
add_header 'Content-Length' 0;
add_header 'Access-Control-Allow-Origin' '*' always;
add_header 'Access-Control-Allow-Methods' 'GET, POST, OPTIONS, HEAD, PUT, DELETE' always;
add_header 'Access-Control-Allow-Headers' '*' always;
add_header 'Access-Control-Max-Age' 86400 always;
add_header 'Content-Length' 0 always;
return 204;
}

rewrite ^/api(.*) /$1 break;
proxy_pass http://geoapi:8000;
proxy_http_version 1.1;
Expand All @@ -39,13 +41,14 @@ http {
location /assets {
max_ranges 0;
expires 30d;
add_header "Access-Control-Allow-Origin" *;
add_header 'Access-Control-Allow-Origin' '*';

# Preflighted requests
if ($request_method = OPTIONS ) {
add_header "Access-Control-Allow-Origin" *;
add_header "Access-Control-Allow-Methods" "GET, POST, OPTIONS, HEAD, PUT, DELETE";
add_header "Access-Control-Allow-Headers" "*";
add_header 'Access-Control-Max-Age' 1728000;
add_header 'Access-Control-Allow-Origin' '*';
add_header 'Access-Control-Allow-Methods' 'GET, POST, OPTIONS, HEAD, PUT, DELETE';
add_header 'Access-Control-Allow-Headers' '*';
add_header 'Access-Control-Max-Age' 86400;
add_header 'Content-Length' 0;
return 204;
}
Expand Down
1 change: 1 addition & 0 deletions geoapi/alembic.ini
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ script_location = migrations

# template used to generate migration files
# file_template = %%(rev)s_%%(slug)s
file_template = %%(year)d%%(month).2d%%(day).2d_%%(hour).2d%%(minute).2d-%%(rev)s_%%(slug)s

# timezone to use when rendering the date
# within the migration file as well as the filename.
Expand Down
8 changes: 4 additions & 4 deletions geoapi/app.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@
from geoapi.settings import settings as app_settings
from geoapi.db import db_session
from geoapi.exceptions import (InvalidGeoJSON, InvalidEXIFData, InvalidCoordinateReferenceSystem,
ObservableProjectAlreadyExists, ApiException, StreetviewAuthException,
ProjectSystemPathWatchFilesAlreadyExists, ApiException, StreetviewAuthException,
StreetviewLimitException, AuthenticationIssue)

import logging
Expand Down Expand Up @@ -48,9 +48,9 @@ def handle_coordinate_reference_system_exception(error: Exception):
return {'message': 'Invalid data, coordinate reference system could not be found'}, 400


@api.errorhandler(ObservableProjectAlreadyExists)
def handle_observable_project_already_exists_exception(error: Exception):
return {'message': 'Conflict, a project for this storage system/path already exists'}, 409
@api.errorhandler(ProjectSystemPathWatchFilesAlreadyExists)
def handle_project_system_path_watch_files_already_exists_exception(error: Exception):
return {'message': 'Conflict, a project watching files for this storage system/path already exists'}, 409


@api.errorhandler(StreetviewAuthException)
Expand Down
13 changes: 9 additions & 4 deletions geoapi/celery_app.py
Original file line number Diff line number Diff line change
@@ -1,7 +1,8 @@
from celery import Celery
from celery.schedules import crontab
from datetime import timedelta
from geoapi.settings import settings


CELERY_CONNECTION_STRING = "amqp://{user}:{pwd}@{hostname}/{vhost}".format(
user=settings.RABBITMQ_USERNAME,
pwd=settings.RABBITMQ_PASSWD,
Expand All @@ -15,8 +16,12 @@
include=['geoapi.tasks'])

app.conf.beat_schedule = {
'refresh_observable_projects': {
'task': 'geoapi.tasks.external_data.refresh_observable_projects',
'schedule': crontab(hour='*', minute='0')
'refresh_projects_watch_content': {
'task': 'geoapi.tasks.external_data.refresh_projects_watch_content',
'schedule': timedelta(hours=1)
},
'refresh_projects_watch_users': {
'task': 'geoapi.tasks.external_data.refresh_projects_watch_users',
'schedule': timedelta(minutes=30)
}
}
4 changes: 2 additions & 2 deletions geoapi/exceptions.py
Original file line number Diff line number Diff line change
Expand Up @@ -14,8 +14,8 @@ class InvalidCoordinateReferenceSystem(Exception):
pass


class ObservableProjectAlreadyExists(Exception):
""" Observable Project already exists for this path"""
class ProjectSystemPathWatchFilesAlreadyExists(Exception):
""" Project with watch_files True already exists for this system path"""
pass


Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,68 @@
"""add_watch_variables_to_project

Revision ID: 968f358e102a
Revises: 4eeeeea72dbc
Create Date: 2024-09-16 18:55:13.685590

"""
from alembic import op
import sqlalchemy as sa
from sqlalchemy.orm import Session
from geoapi.log import logger


# revision identifiers, used by Alembic.
revision = '968f358e102a'
down_revision = '4eeeeea72dbc'
branch_labels = None
depends_on = None


def upgrade():
# Add new columns
op.add_column('projects', sa.Column('watch_content', sa.Boolean(), nullable=True))
op.add_column('projects', sa.Column('watch_users', sa.Boolean(), nullable=True))

bind = op.get_bind()
session = Session(bind=bind)

try:
# Query all projects and their related observable data projects in order
# to set the watch_content and watch_users
projects_query = sa.text("""
SELECT p.id, odp.id as odp_id, odp.watch_content
FROM projects p
LEFT JOIN observable_data_projects odp ON p.id = odp.project_id
""")
results = session.execute(projects_query)

# Update projects based on the query results
for project_id, odp_id, odp_watch_content in results:
update_query = sa.text("""
UPDATE projects
SET watch_content = :watch_content, watch_users = :watch_users
WHERE id = :project_id
""")
session.execute(update_query, {
'watch_content': odp_watch_content if odp_id is not None else False,
'watch_users': odp_id is not None,
'project_id': project_id
})
session.commit()
logger.info(f"Data migration of project/observable completed successfully")

except Exception as e:
session.rollback()
logger.exception(f"An error occurred during project/observable data migration: {str(e)}")
raise
finally:
session.close()

# Drop the unique constraint
op.drop_constraint('observable_data_projects_system_id_path_key', 'observable_data_projects')


def downgrade():
op.drop_column('projects', 'watch_users')
op.drop_column('projects', 'watch_content')
op.create_unique_constraint('observable_data_projects_system_id_path_key', 'observable_data_projects', ['system_id', 'path'])
6 changes: 2 additions & 4 deletions geoapi/models/observable_data.py
Original file line number Diff line number Diff line change
@@ -1,17 +1,15 @@
from sqlalchemy import (
Column, Integer, String, Boolean,
ForeignKey, DateTime, UniqueConstraint
ForeignKey, DateTime
)
from sqlalchemy.orm import relationship
from sqlalchemy.sql import func
from geoapi.db import Base


# Deprecated; Replaced by watch_user and watch_content of Project table. See https://tacc-main.atlassian.net/browse/WG-377
class ObservableDataProject(Base):
__tablename__ = 'observable_data_projects'
__table_args__ = (
UniqueConstraint('system_id', 'path', name="unique_system_id_path"),
)
id = Column(Integer, primary_key=True)
project_id = Column(ForeignKey('projects.id', ondelete="CASCADE", onupdate="CASCADE"), index=True)
created_date = Column(DateTime(timezone=True), server_default=func.now())
Expand Down
18 changes: 15 additions & 3 deletions geoapi/models/project.py
Original file line number Diff line number Diff line change
Expand Up @@ -29,10 +29,11 @@ class Project(Base):
id = Column(Integer, primary_key=True)
uuid = Column(UUID(as_uuid=True), default=uuid.uuid4, nullable=False)
tenant_id = Column(String, nullable=False)
# Project system_id/system_path really not used except for analytics.
# This could be improved; see https://jira.tacc.utexas.edu/browse/WG-185
# associated tapis system id
system_id = Column(String, nullable=True)
# associated tapis system path
system_path = Column(String, nullable=True)
# associated tapis system file
system_file = Column(String, nullable=True)
name = Column(String, nullable=False)
description = Column(String)
Expand All @@ -47,5 +48,16 @@ class Project(Base):
overlaps="project,project_users")
point_clouds = relationship('PointCloud', cascade="all, delete-orphan")

# watch content of tapis directory location (system_id and system_path)
watch_content = Column(Boolean, default=False)

# watch user of tapis system (system_id)
watch_users = Column(Boolean, default=False)

def __repr__(self):
return '<Project(id={})>'.format(self.id)
return f"<Project(id={self.id}, uuid={self.uuid}, tenant_id='{self.tenant_id}', " \
f"system_id='{self.system_id}', system_path='{self.system_path}', " \
f"system_file='{self.system_file}', name='{self.name}', " \
f"description='{self.description}', public={self.public}, " \
f"created={self.created}, updated={self.updated}, " \
f"watch_content={self.watch_content}, watch_users={self.watch_users})>"
8 changes: 7 additions & 1 deletion geoapi/routes/projects.py
Original file line number Diff line number Diff line change
Expand Up @@ -60,6 +60,12 @@
'watch_users': fields.Boolean()
})

project_update_payload = api.model('Project', {
'name': fields.String(),
'description': fields.String(),
'public': fields.Boolean(),
})

project_response = api.model('ProjectResponse', {
'id': fields.Integer(),
'uuid': fields.String(),
Expand Down Expand Up @@ -225,14 +231,14 @@ def delete(self, projectId: int):

@api.doc(id="updateProject",
description="Update metadata about a project")
@api.expect(project_update_payload)
@api.marshal_with(project_response)
@project_permissions
def put(self, projectId: int):
u = request.current_user
logger.info("Update project:{} for user:{}".format(projectId,
u.username))
return ProjectsService.update(db_session,
user=u,
projectId=projectId,
data=api.payload)

Expand Down
Loading
Loading