python black formatting; flake8 on everything

.flake8

@@ -1,11 +1,10 @@
 [flake8]
+# E203: Whitespace before ':'.
+# H101: Use TODO(NAME)
+# W503: line break before binary operator.
+ignore  = E203, H101, W503
 max-line-length = 140
-exclude = 
-    geoapi/migrations/
+exclude =
     geoapi/models/__init__.py
     geoapi/tasks/__init__.py
     geoapi/schemas/__init__.py
-    # Files below are tied with https://jira.tacc.utexas.edu/browse/DES-2267
-    geoapi/initdb.py
-    geoapi/utils/streetview.py
-    geoapi/services/streetview.py

.github/workflows/main.yml

@@ -2,9 +2,9 @@ name: CI
 
 on:
   push:
-    branches: [ main ]
+    branches: [main]
   pull_request:
-    branches: [ '**' ]
+    branches: ["**"]
 jobs:
   Linting:
     runs-on: ubuntu-latest
@@ -14,10 +14,14 @@ jobs:
         uses: actions/setup-python@v4
         with:
           python-version: 3.9
-      - name: Install flake8
-        run: pip install flake8
+      - name: Install flake8 and black
+        run: pip install flake8 black
       - name: Run flake8
         run: flake8 geoapi
+      - name: Formatting with black
+        run: |
+          black --check .
+
   Geoapi_Unit_Tests:
     runs-on: ubuntu-latest
     env:
@@ -31,40 +35,40 @@ jobs:
           POSTGRES_PASSWORD: dev
           POSTGRES_DB: test
         ports:
-          -  5432:5432
+          - 5432:5432
         options: >-
           --health-cmd pg_isready
           --health-interval 10s
           --health-timeout 5s
           --health-retries 5
     steps:
-    - uses: actions/checkout@v3
-    - name: Setup Python
-      uses: actions/setup-python@v4
-      with:
-        python-version: 3.9
-    - name: Install ffmpeg
-      run: |
-        sudo apt-get update
-        sudo apt-get install -y ffmpeg
-    - name: Setup Poetry
-      uses: abatilo/[email protected]
-      with:
-        poetry-version: 1.8.3
-    - uses: actions/cache@v1
-      with:
-        path: ~/.cache/pypoetry/virtualenvs
-        key: ${{ runner.os }}-poetry-${{ hashFiles('**/poetry.lock') }}
-        restore-keys: |
-          ${{ runner.os }}-poetry-
-    - name: Install Python Packages
-      run: |
-        cd devops
-        poetry install
-    - name: Run server-side unit tests
-      run: |
-        cd devops
-        poetry run pytest ../geoapi
+      - uses: actions/checkout@v3
+      - name: Setup Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: 3.9
+      - name: Install ffmpeg
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y ffmpeg
+      - name: Setup Poetry
+        uses: abatilo/[email protected]
+        with:
+          poetry-version: 1.8.3
+      - uses: actions/cache@v1
+        with:
+          path: ~/.cache/pypoetry/virtualenvs
+          key: ${{ runner.os }}-poetry-${{ hashFiles('**/poetry.lock') }}
+          restore-keys: |
+            ${{ runner.os }}-poetry-
+      - name: Install Python Packages
+        run: |
+          cd devops
+          poetry install
+      - name: Run server-side unit tests
+        run: |
+          cd devops
+          poetry run pytest ../geoapi
   Workers_Unit_Tests:
     runs-on: ubuntu-latest
     services:
@@ -82,15 +86,15 @@ jobs:
           --health-timeout 5s
           --health-retries 5
     steps:
-    - uses: actions/checkout@v3
-    - uses: docker/login-action@v1
-      with:
-        password: ${{ secrets.DOCKER_PASSWORD }}
-        username: ${{ secrets.DOCKER_USER  }}
-    - name: Build worker image
-      run: |
-        docker pull taccaci/geoapi-workers:latest
-        docker build --cache-from taccaci/geoapi-workers:latest -t taccaci/geoapi-workers:latest -f devops/Dockerfile.worker .
-    - name: Run worker test
-      run: |
-        docker run --network="host" -e APP_ENV='testing' -e DB_HOST='localhost'  taccaci/geoapi-workers:latest pytest -m "worker"
+      - uses: actions/checkout@v3
+      - uses: docker/login-action@v1
+        with:
+          password: ${{ secrets.DOCKER_PASSWORD }}
+          username: ${{ secrets.DOCKER_USER  }}
+      - name: Build worker image
+        run: |
+          docker pull taccaci/geoapi-workers:latest
+          docker build --cache-from taccaci/geoapi-workers:latest -t taccaci/geoapi-workers:latest -f devops/Dockerfile.worker .
+      - name: Run worker test
+        run: |
+          docker run --network="host" -e APP_ENV='testing' -e DB_HOST='localhost'  taccaci/geoapi-workers:latest pytest -m "worker"

.gitignore

@@ -133,3 +133,5 @@ client/swagger-codegen/
 
 # celerybeat schedule files
 celerybeat-schedule.*
+
+.vscode/

client_use_examples/load_test_point_cloud_import_prod.py

@@ -3,30 +3,49 @@
 import concurrent.futures
 import os
 
-TOKEN = os.environ['TOKEN']
-EXAMPLE_FILE = {"files": [{"path": "/3_Deliverables/Safeway Supermarket/Pointclouds/TLS/LAZ_combined/Safeway_combined_3cm_20190329.laz", "system": "project-159846449346309655-242ac119-0001-012"}]}
+TOKEN = os.environ["TOKEN"]
+EXAMPLE_FILE = {
+    "files": [
+        {
+            "path": "/3_Deliverables/Safeway Supermarket/Pointclouds/TLS/LAZ_combined/Safeway_combined_3cm_20190329.laz",
+            "system": "project-159846449346309655-242ac119-0001-012",
+        }
+    ]
+}
 configuration = geoapi_client.Configuration()
 configuration.host = "https://agave.designsafe-ci.org/geo/v2"
-configuration.api_key_prefix['Authorization'] = 'Bearer'
-configuration.api_key['Authorization'] = TOKEN
+configuration.api_key_prefix["Authorization"] = "Bearer"
+configuration.api_key["Authorization"] = TOKEN
 
 api_client = geoapi_client.ApiClient(configuration)
 api_instance = geoapi_client.ProjectsApi(api_client=api_client)
 
+
 def create_point_cloud_and_import_file(project_id, index):
-    point_cloud = api_instance.add_point_cloud(project_id=project_id, payload={"description": f"point_cloud{index}", "files_info": "dummy" })
-    point_cloud_importing = api_instance.import_point_cloud_file_from_tapis(project_id=project.id, point_cloud_id=point_cloud.id, payload=EXAMPLE_FILE)
+    point_cloud = api_instance.add_point_cloud(
+        project_id=project_id,
+        payload={"description": f"point_cloud{index}", "files_info": "dummy"},
+    )
+    point_cloud_importing = api_instance.import_point_cloud_file_from_tapis(
+        project_id=project.id, point_cloud_id=point_cloud.id, payload=EXAMPLE_FILE
+    )
     print(f"submitted: {index} {point_cloud_importing}")
     return index, point_cloud_importing
 
+
 try:
     test_id = "Testing multiple point cloud load"
-    project = api_instance.create_project(payload={"project": {"name": f"My project {test_id}"}})
+    project = api_instance.create_project(
+        payload={"project": {"name": f"My project {test_id}"}}
+    )
     project_id = project.id
     workers = 20
-    number_of_point_clouds=100
+    number_of_point_clouds = 100
     with concurrent.futures.ThreadPoolExecutor(max_workers=workers) as executor:
-        importing_point_cloud_futures = [executor.submit(create_point_cloud_and_import_file, project_id, index) for index in list(range(1, number_of_point_clouds))]
+        importing_point_cloud_futures = [
+            executor.submit(create_point_cloud_and_import_file, project_id, index)
+            for index in list(range(1, number_of_point_clouds))
+        ]
 
         results = []
         for future in concurrent.futures.as_completed(importing_point_cloud_futures):

geoapi/app.py

@@ -1,15 +1,21 @@
-
 from flask import Flask
 from geoapi.routes import api
 from geoapi.settings import settings as app_settings
 from geoapi.db import db_session
-from geoapi.exceptions import (InvalidGeoJSON, InvalidEXIFData, InvalidCoordinateReferenceSystem,
-                               ProjectSystemPathWatchFilesAlreadyExists, ApiException, StreetviewAuthException,
-                               StreetviewLimitException, AuthenticationIssue)
+from geoapi.exceptions import (
+    InvalidGeoJSON,
+    InvalidEXIFData,
+    InvalidCoordinateReferenceSystem,
+    ProjectSystemPathWatchFilesAlreadyExists,
+    ApiException,
+    StreetviewAuthException,
+    StreetviewLimitException,
+    AuthenticationIssue,
+)
 
 import logging
 
-logging.basicConfig(format='%(asctime)s %(message)s', level=logging.INFO)
+logging.basicConfig(format="%(asctime)s %(message)s", level=logging.INFO)
 
 
 app = Flask(__name__)
@@ -19,53 +25,49 @@
 
 @api.errorhandler(InvalidGeoJSON)
 def handle_geojson_exception(error: Exception):
-    '''Return a custom message and 400 status code'''
-    return ({
-        "status": "error",
-        "version": "0.1",
-        "message": str(error)
-    }, 400)
+    """Return a custom message and 400 status code"""
+    return ({"status": "error", "version": "0.1", "message": str(error)}, 400)
 
 
 @api.errorhandler(ApiException)
 def handle_api_exception(error: Exception):
-    '''Return a custom message and 400 status code'''
-    return {
-        "status": "error",
-        "version": "0.1",
-        "message": str(error)
-    }, 400
+    """Return a custom message and 400 status code"""
+    return {"status": "error", "version": "0.1", "message": str(error)}, 400
 
 
 @api.errorhandler(InvalidEXIFData)
 def handle_exif_exception(error: Exception):
-    '''Return a custom message and 400 status code'''
-    return {'message': 'Invalid EXIF data, geolocation could not be found'}, 400
+    """Return a custom message and 400 status code"""
+    return {"message": "Invalid EXIF data, geolocation could not be found"}, 400
 
 
 @api.errorhandler(InvalidCoordinateReferenceSystem)
 def handle_coordinate_reference_system_exception(error: Exception):
-    return {'message': 'Invalid data, coordinate reference system could not be found'}, 400
+    return {
+        "message": "Invalid data, coordinate reference system could not be found"
+    }, 400
 
 
 @api.errorhandler(ProjectSystemPathWatchFilesAlreadyExists)
 def handle_project_system_path_watch_files_already_exists_exception(error: Exception):
-    return {'message': 'Conflict, a project watching files for this storage system/path already exists'}, 409
+    return {
+        "message": "Conflict, a project watching files for this storage system/path already exists"
+    }, 409
 
 
 @api.errorhandler(StreetviewAuthException)
 def handle_streetview_auth_exception(error: Exception):
-    return {'message': 'Not logged in to streetview service'}, 401
+    return {"message": "Not logged in to streetview service"}, 401
 
 
 @api.errorhandler(StreetviewLimitException)
 def handle_streetview_limit_exception(error: Exception):
-    return {'message': 'Exceed concurrent streetview publish limit'}, 403
+    return {"message": "Exceed concurrent streetview publish limit"}, 403
 
 
 @api.errorhandler(AuthenticationIssue)
 def handle_authentication_issue_exception(error: AuthenticationIssue):
-    return {'message': error.message}, 400
+    return {"message": error.message}, 400
 
 
 # ensure SQLAlchemy sessions are properly closed at the end of each request.

geoapi/celery_app.py

@@ -7,29 +7,28 @@
     user=settings.RABBITMQ_USERNAME,
     pwd=settings.RABBITMQ_PASSWD,
     hostname=settings.RABBITMQ_HOSTNAME,
-    vhost=settings.RABBITMQ_VHOST
+    vhost=settings.RABBITMQ_VHOST,
 )
 
-app = Celery('hello',
-             backend='rpc',
-             broker=CELERY_CONNECTION_STRING,
-             include=['geoapi.tasks'])
+app = Celery(
+    "hello", backend="rpc", broker=CELERY_CONNECTION_STRING, include=["geoapi.tasks"]
+)
 
 # Define the queues
 app.conf.task_queues = {
-    'default': {'exchange': 'default', 'routing_key': 'default'},
-    'heavy': {'exchange': 'heavy', 'routing_key': 'heavy'}
+    "default": {"exchange": "default", "routing_key": "default"},
+    "heavy": {"exchange": "heavy", "routing_key": "heavy"},
 }
 
-app.conf.task_default_queue = 'default'
+app.conf.task_default_queue = "default"
 
 app.conf.beat_schedule = {
-    'refresh_projects_watch_content': {
-        'task': 'geoapi.tasks.external_data.refresh_projects_watch_content',
-        'schedule': timedelta(hours=1)
+    "refresh_projects_watch_content": {
+        "task": "geoapi.tasks.external_data.refresh_projects_watch_content",
+        "schedule": timedelta(hours=1),
+    },
+    "refresh_projects_watch_users": {
+        "task": "geoapi.tasks.external_data.refresh_projects_watch_users",
+        "schedule": timedelta(minutes=30),
     },
-    'refresh_projects_watch_users': {
-        'task': 'geoapi.tasks.external_data.refresh_projects_watch_users',
-        'schedule': timedelta(minutes=30)
-    }
 }

geoapi/custom/__init__.py

@@ -3,8 +3,12 @@
 
 custom_system_user_retrieval = {"DESIGNSAFE": get_system_users}
 
-custom_on_project_creation = {"DESIGNSAFE": on_project_creation,
-                              "PORTALS": on_project_creation}  # TODO_TAPISV3 using portals for testing
+custom_on_project_creation = {
+    "DESIGNSAFE": on_project_creation,
+    "PORTALS": on_project_creation,
+}  # TODO_TAPISV3 using portals for testing
 
-custom_on_project_deletion = {"DESIGNSAFE": on_project_deletion,
-                              "PORTALS": on_project_deletion}  # TODO_TAPISV3 using portals for testing
+custom_on_project_deletion = {
+    "DESIGNSAFE": on_project_deletion,
+    "PORTALS": on_project_deletion,
+}  # TODO_TAPISV3 using portals for testing