diff --git a/Makefile b/Makefile index 3a32bd3..46b52fc 100644 --- a/Makefile +++ b/Makefile @@ -9,5 +9,9 @@ install: publish: poetry run mono publish +format: + poetry run isort . + poetry run black . + test: poetry run pytest -s -x --failed-first \ No newline at end of file diff --git a/app-frame/macrostrat/app_frame/__init__.py b/app-frame/macrostrat/app_frame/__init__.py index a471258..78d067a 100644 --- a/app-frame/macrostrat/app_frame/__init__.py +++ b/app-frame/macrostrat/app_frame/__init__.py @@ -1,4 +1,4 @@ -from .core import Application from .compose import compose -from .subsystems import SubsystemManager, Subsystem, SubsystemError +from .core import Application from .exc import ApplicationError +from .subsystems import Subsystem, SubsystemError, SubsystemManager diff --git a/app-frame/macrostrat/app_frame/compose.py b/app-frame/macrostrat/app_frame/compose.py index 344d986..b7522c5 100644 --- a/app-frame/macrostrat/app_frame/compose.py +++ b/app-frame/macrostrat/app_frame/compose.py @@ -1,6 +1,7 @@ -from macrostrat.utils import cmd, get_logger from rich.console import Console +from macrostrat.utils import cmd, get_logger + console = Console() log = get_logger(__name__) diff --git a/app-frame/macrostrat/app_frame/control_command.py b/app-frame/macrostrat/app_frame/control_command.py index eb79d30..a0b1e23 100644 --- a/app-frame/macrostrat/app_frame/control_command.py +++ b/app-frame/macrostrat/app_frame/control_command.py @@ -6,11 +6,12 @@ import click import typer from click import Group -from macrostrat.utils import get_logger from typer import Context, Typer from typer.core import TyperGroup from typer.models import TyperInfo +from macrostrat.utils import get_logger + from .compose import check_status, compose from .core import Application from .follow_logs import Result, command_stream, follow_logs diff --git a/app-frame/macrostrat/app_frame/core.py b/app-frame/macrostrat/app_frame/core.py index d301df9..feb52b2 100644 --- a/app-frame/macrostrat/app_frame/core.py +++ b/app-frame/macrostrat/app_frame/core.py @@ -1,11 +1,12 @@ -from rich.console import Console -from typing import Optional, Callable -from macrostrat.utils import setup_stderr_logs, get_logger import logging from os import environ from pathlib import Path +from typing import Callable, Optional + from dotenv import load_dotenv +from rich.console import Console +from macrostrat.utils import get_logger, setup_stderr_logs log = get_logger(__name__) diff --git a/app-frame/macrostrat/app_frame/follow_logs.py b/app-frame/macrostrat/app_frame/follow_logs.py index 5931e31..029f552 100644 --- a/app-frame/macrostrat/app_frame/follow_logs.py +++ b/app-frame/macrostrat/app_frame/follow_logs.py @@ -3,11 +3,11 @@ import sys import termios from contextlib import contextmanager +from datetime import datetime from enum import Enum from subprocess import Popen from time import sleep from typing import Generator -from datetime import datetime from macrostrat.utils import get_logger diff --git a/app-frame/macrostrat/app_frame/subsystems/__init__.py b/app-frame/macrostrat/app_frame/subsystems/__init__.py index 8cafcd0..38488a0 100644 --- a/app-frame/macrostrat/app_frame/subsystems/__init__.py +++ b/app-frame/macrostrat/app_frame/subsystems/__init__.py @@ -1,14 +1,14 @@ -from toposort import toposort_flatten -from ..core import ApplicationBase +from typing import Optional + from packaging.specifiers import InvalidSpecifier, SpecifierSet from packaging.version import Version -from typing import Optional +from toposort import toposort_flatten from macrostrat.utils.logs import get_logger +from ..core import ApplicationBase from .defs import Subsystem, SubsystemError - log = get_logger(__name__) diff --git a/app-frame/test_app/__init__.py b/app-frame/test_app/__init__.py index 725d059..4860b54 100644 --- a/app-frame/test_app/__init__.py +++ b/app-frame/test_app/__init__.py @@ -1,6 +1,6 @@ from pathlib import Path -from macrostrat.app_frame import Application +from macrostrat.app_frame import Application APP_ROOT = Path(__file__).parent diff --git a/conftest.py b/conftest.py index 5da2854..ddc9691 100644 --- a/conftest.py +++ b/conftest.py @@ -1,11 +1,13 @@ -from pytest import fixture from os import environ -from macrostrat.dinosaur.upgrade_cluster.utils import database_cluster, get_unused_port -from dotenv import load_dotenv + from docker.client import DockerClient +from dotenv import load_dotenv +from pytest import fixture from sqlalchemy import create_engine from sqlalchemy.exc import OperationalError +from macrostrat.dinosaur.upgrade_cluster.utils import database_cluster, get_unused_port + load_dotenv() diff --git a/database/macrostrat/database/__init__.py b/database/macrostrat/database/__init__.py index 0c64f4d..fd466bc 100644 --- a/database/macrostrat/database/__init__.py +++ b/database/macrostrat/database/__init__.py @@ -2,23 +2,17 @@ from contextlib import contextmanager from typing import Optional -from sqlalchemy import create_engine, inspect, MetaData, text -from sqlalchemy.orm import sessionmaker, scoped_session, Session +from sqlalchemy import MetaData, create_engine, inspect, text from sqlalchemy.exc import IntegrityError -from macrostrat.utils import get_logger from sqlalchemy.ext.compiler import compiles +from sqlalchemy.orm import Session, scoped_session, sessionmaker from sqlalchemy.sql.expression import Insert +from macrostrat.utils import get_logger -from .utils import ( - run_sql, - get_or_create, - reflect_table, - get_dataframe, -) from .mapper import DatabaseMapper -from .postgresql import prefix_inserts, on_conflict # noqa - +from .postgresql import on_conflict, prefix_inserts # noqa +from .utils import get_dataframe, get_or_create, reflect_table, run_sql metadata = MetaData() diff --git a/database/macrostrat/database/mapper/__init__.py b/database/macrostrat/database/mapper/__init__.py index 463d3b0..3cf2292 100644 --- a/database/macrostrat/database/mapper/__init__.py +++ b/database/macrostrat/database/mapper/__init__.py @@ -1,20 +1,21 @@ from distutils.log import warn -from macrostrat.database.utils import reflect_table -from sqlalchemy.ext.automap import generate_relationship -from macrostrat.utils.logs import get_logger from warnings import warn # Drag in geographic types for database reflection -from geoalchemy2 import Geometry, Geography +from geoalchemy2 import Geography, Geometry +from sqlalchemy.ext.automap import generate_relationship + +from macrostrat.database.utils import reflect_table +from macrostrat.utils.logs import get_logger from .cache import DatabaseModelCache from .utils import ( ModelCollection, TableCollection, - classname_for_table, _classname_for_table, - name_for_scalar_relationship, + classname_for_table, name_for_collection_relationship, + name_for_scalar_relationship, ) log = get_logger(__name__) diff --git a/database/macrostrat/database/mapper/cache.py b/database/macrostrat/database/mapper/cache.py index 11de3eb..5171b2b 100644 --- a/database/macrostrat/database/mapper/cache.py +++ b/database/macrostrat/database/mapper/cache.py @@ -1,8 +1,10 @@ +from os import makedirs, path +from pickle import dump, load + from sqlalchemy.ext.automap import automap_base -from os import path, makedirs + from macrostrat.utils.logs import get_logger -from sqlalchemy.ext.automap import automap_base -from pickle import load, dump + from .base import ModelHelperMixins log = get_logger(__name__) diff --git a/database/macrostrat/database/postgresql.py b/database/macrostrat/database/postgresql.py index 3fddd4b..50c12d2 100644 --- a/database/macrostrat/database/postgresql.py +++ b/database/macrostrat/database/postgresql.py @@ -2,15 +2,13 @@ from contextlib import contextmanager from contextvars import ContextVar +from typing import TYPE_CHECKING +import psycopg2 +from sqlalchemy.dialects import postgresql from sqlalchemy.exc import CompileError from sqlalchemy.ext.compiler import compiles from sqlalchemy.sql.expression import Insert, text -from sqlalchemy.dialects import postgresql -import psycopg2 - - -from typing import TYPE_CHECKING if TYPE_CHECKING: from ..database import Database diff --git a/database/macrostrat/database/utils.py b/database/macrostrat/database/utils.py index 2c0bb81..e4320f1 100644 --- a/database/macrostrat/database/utils.py +++ b/database/macrostrat/database/utils.py @@ -1,22 +1,28 @@ +from contextlib import contextmanager +from pathlib import Path +from re import search +from time import sleep +from typing import IO, Union +from warnings import warn + from click import echo, secho -from sqlalchemy.exc import ProgrammingError, IntegrityError, InternalError -from sqlparse import split, format -from sqlalchemy.sql.elements import TextClause, ClauseElement +from psycopg2.sql import SQL, Composable, Composed +from sqlalchemy import MetaData, create_engine, text +from sqlalchemy.engine import Connection, Engine +from sqlalchemy.exc import ( + IntegrityError, + InternalError, + InvalidRequestError, + ProgrammingError, +) from sqlalchemy.orm import sessionmaker -from sqlalchemy.engine import Engine, Connection from sqlalchemy.schema import Table -from sqlalchemy import MetaData, create_engine, text -from contextlib import contextmanager +from sqlalchemy.sql.elements import ClauseElement, TextClause from sqlalchemy_utils import create_database, database_exists, drop_database -from sqlalchemy.exc import InvalidRequestError +from sqlparse import format, split + from macrostrat.utils import cmd, get_logger -from time import sleep -from typing import Union, IO -from pathlib import Path -from warnings import warn -from psycopg2.sql import SQL, Composable, Composed -from re import search -from macrostrat.utils import get_logger + from .postgresql import _setup_psycopg2_wait_callback log = get_logger(__name__) diff --git a/database/test-scripts/test-long-running-query b/database/test-scripts/test-long-running-query index 26d7635..24e37bc 100755 --- a/database/test-scripts/test-long-running-query +++ b/database/test-scripts/test-long-running-query @@ -3,10 +3,11 @@ Run a long-running query to test if we can interrupt it. """ -from macrostrat.database import Database from sys import argv, exit + from sqlalchemy.exc import OperationalError +from macrostrat.database import Database db_conn = argv[1] print(f"Connecting to {db_conn}") diff --git a/database/test_database.py b/database/test_database.py index 2a6a024..2c32242 100644 --- a/database/test_database.py +++ b/database/test_database.py @@ -1,17 +1,17 @@ # Skeletal testing file +from os import environ from pathlib import Path -from pytest import fixture + from dotenv import load_dotenv from psycopg2.sql import SQL, Identifier, Literal, Placeholder +from pytest import fixture, mark, raises, warns from sqlalchemy.exc import ProgrammingError from sqlalchemy.sql import text -from macrostrat.utils import relative_path, get_logger from macrostrat.database import Database, run_sql -from macrostrat.database.utils import temp_database, infer_is_sql_text from macrostrat.database.postgresql import table_exists -from pytest import warns, raises, mark -from os import environ +from macrostrat.database.utils import infer_is_sql_text, temp_database +from macrostrat.utils import get_logger, relative_path load_dotenv() @@ -215,6 +215,7 @@ def test_close_connection(conn): """ import threading + from psycopg2.extensions import QueryCanceledError from sqlalchemy.exc import DBAPIError @@ -240,8 +241,8 @@ def test_sigint_cancel(db): by sending a SIGINT. """ - import subprocess import signal + import subprocess import time db_url = str(db.engine.url) diff --git a/dinosaur/macrostrat/dinosaur/__init__.py b/dinosaur/macrostrat/dinosaur/__init__.py index a74d195..5be14d0 100644 --- a/dinosaur/macrostrat/dinosaur/__init__.py +++ b/dinosaur/macrostrat/dinosaur/__init__.py @@ -1,25 +1,21 @@ import os import sys from contextlib import contextmanager, redirect_stdout +from typing import Callable -from sqlalchemy.exc import ProgrammingError, IntegrityError, DataError -from schemainspect import get_inspector +import docker from migra import Migration from migra.statements import check_for_drop -from sqlalchemy import text from rich import print -from typing import Callable -import docker +from schemainspect import get_inspector +from sqlalchemy import text +from sqlalchemy.exc import DataError, IntegrityError, ProgrammingError -from macrostrat.utils import get_logger, cmd from macrostrat.database import Database -from macrostrat.database.utils import ( - run_sql, - temp_database, - connection_args, -) -from .upgrade_cluster.utils import wait_for_cluster, wait_for_ready +from macrostrat.database.utils import connection_args, run_sql, temp_database +from macrostrat.utils import cmd, get_logger +from .upgrade_cluster.utils import wait_for_cluster, wait_for_ready log = get_logger(__name__) diff --git a/dinosaur/macrostrat/dinosaur/upgrade_cluster/__init__.py b/dinosaur/macrostrat/dinosaur/upgrade_cluster/__init__.py index 931ca91..f65e13a 100644 --- a/dinosaur/macrostrat/dinosaur/upgrade_cluster/__init__.py +++ b/dinosaur/macrostrat/dinosaur/upgrade_cluster/__init__.py @@ -1,19 +1,21 @@ +from typing import List + from docker.client import DockerClient -from macrostrat.utils import get_logger from rich.console import Console -from typing import List +from macrostrat.utils import get_logger + +from .describe import ( + check_database_cluster_version, + check_database_exists, + count_database_tables, +) +from .restore import pg_restore from .utils import ( database_cluster, ensure_empty_docker_volume, - replace_docker_volume, get_unused_port, -) -from .restore import pg_restore -from .describe import ( - check_database_exists, - count_database_tables, - check_database_cluster_version, + replace_docker_volume, ) log = get_logger(__name__) diff --git a/dinosaur/macrostrat/dinosaur/upgrade_cluster/describe.py b/dinosaur/macrostrat/dinosaur/upgrade_cluster/describe.py index ac0a4d7..240b3e9 100644 --- a/dinosaur/macrostrat/dinosaur/upgrade_cluster/describe.py +++ b/dinosaur/macrostrat/dinosaur/upgrade_cluster/describe.py @@ -1,8 +1,9 @@ -from docker.models.containers import Container +from pathlib import Path from subprocess import CalledProcessError -from docker.errors import ContainerError + from docker.client import DockerClient -from pathlib import Path +from docker.errors import ContainerError +from docker.models.containers import Container from macrostrat.utils import get_logger diff --git a/dinosaur/macrostrat/dinosaur/upgrade_cluster/restore.py b/dinosaur/macrostrat/dinosaur/upgrade_cluster/restore.py index e5a13ad..1ac8464 100644 --- a/dinosaur/macrostrat/dinosaur/upgrade_cluster/restore.py +++ b/dinosaur/macrostrat/dinosaur/upgrade_cluster/restore.py @@ -1,8 +1,8 @@ import asyncio + from docker.models.containers import Container from rich.console import Console - console = Console() diff --git a/dinosaur/macrostrat/dinosaur/upgrade_cluster/utils.py b/dinosaur/macrostrat/dinosaur/upgrade_cluster/utils.py index d54e8ae..ff559f2 100644 --- a/dinosaur/macrostrat/dinosaur/upgrade_cluster/utils.py +++ b/dinosaur/macrostrat/dinosaur/upgrade_cluster/utils.py @@ -1,14 +1,15 @@ +import socket import time from contextlib import contextmanager -import socket +from typing import Mapping, Optional import docker from docker.client import DockerClient from docker.models.containers import Container -from macrostrat.utils import get_logger from sqlalchemy import create_engine from sqlalchemy.exc import OperationalError -from typing import Optional, Mapping + +from macrostrat.utils import get_logger log = get_logger(__name__) diff --git a/dinosaur/tests/test_upgrade_cluster.py b/dinosaur/tests/test_upgrade_cluster.py index 1d59043..21236c3 100644 --- a/dinosaur/tests/test_upgrade_cluster.py +++ b/dinosaur/tests/test_upgrade_cluster.py @@ -1,33 +1,34 @@ """ PostgreSQL/PostGIS cluster upgrade tests reliant onto Docker. """ -from pytest import fixture +import random +import time +from os import environ +from pathlib import Path + from docker.client import DockerClient +from pytest import fixture from sqlalchemy import create_engine, inspect, text + from macrostrat.database import Database from macrostrat.database.utils import run_sql_file +from macrostrat.dinosaur import create_schema_clone, dump_schema from macrostrat.utils import get_logger -from macrostrat.dinosaur import dump_schema, create_schema_clone -from pathlib import Path -from os import environ -import time -import random log = get_logger(__name__) random_hex = lambda: "%08x" % random.randrange(16**8) -from macrostrat.dinosaur.upgrade_cluster.utils import ( - database_cluster, - get_unused_port, - ensure_empty_docker_volume, -) - from macrostrat.dinosaur.upgrade_cluster import ( - upgrade_database_cluster, default_version_images, + upgrade_database_cluster, ) from macrostrat.dinosaur.upgrade_cluster.describe import check_database_cluster_version +from macrostrat.dinosaur.upgrade_cluster.utils import ( + database_cluster, + ensure_empty_docker_volume, + get_unused_port, +) root_dir = Path(__file__).parent fixtures = root_dir / "fixtures" diff --git a/package-tools/macrostrat/package_tools/__init__.py b/package-tools/macrostrat/package_tools/__init__.py index 991580e..a801f9c 100644 --- a/package-tools/macrostrat/package_tools/__init__.py +++ b/package-tools/macrostrat/package_tools/__init__.py @@ -1,4 +1,5 @@ from typer import Typer + from .install import install_packages from .publish import publish_packages diff --git a/package-tools/macrostrat/package_tools/dependencies.py b/package-tools/macrostrat/package_tools/dependencies.py index 9813141..ed7f2d8 100644 --- a/package-tools/macrostrat/package_tools/dependencies.py +++ b/package-tools/macrostrat/package_tools/dependencies.py @@ -1,4 +1,5 @@ from pathlib import Path + from toml import load diff --git a/package-tools/macrostrat/package_tools/install.py b/package-tools/macrostrat/package_tools/install.py index e6dd07a..5d70358 100644 --- a/package-tools/macrostrat/package_tools/install.py +++ b/package-tools/macrostrat/package_tools/install.py @@ -1,8 +1,10 @@ from os import environ -from macrostrat.utils import cmd from pathlib import Path + from rich import print +from macrostrat.utils import cmd + from .dependencies import get_local_dependencies, load_poetry_config diff --git a/package-tools/macrostrat/package_tools/publish.py b/package-tools/macrostrat/package_tools/publish.py index 1e66e19..8a1d3a4 100644 --- a/package-tools/macrostrat/package_tools/publish.py +++ b/package-tools/macrostrat/package_tools/publish.py @@ -1,13 +1,16 @@ #!/usr/bin/env python from os import environ -from macrostrat.utils.shell import git_has_changes from pathlib import Path + import requests -from toml import load from rich import print -from .dependencies import get_local_dependencies, load_poetry_config +from toml import load + from macrostrat.utils import cmd, working_directory +from macrostrat.utils.shell import git_has_changes + +from .dependencies import get_local_dependencies, load_poetry_config def prepare_module(fp: Path): diff --git a/poetry.lock b/poetry.lock index e0958f7..4846c56 100644 --- a/poetry.lock +++ b/poetry.lock @@ -323,6 +323,20 @@ files = [ {file = "iniconfig-2.0.0.tar.gz", hash = "sha256:2d91e135bf72d31a410b17c16da610a82cb55f6b0477d1a902134b24a455b8b3"}, ] +[[package]] +name = "isort" +version = "5.13.2" +description = "A Python utility / library to sort Python imports." +optional = false +python-versions = ">=3.8.0" +files = [ + {file = "isort-5.13.2-py3-none-any.whl", hash = "sha256:8ca5e72a8d85860d5a3fa69b8745237f2939afe12dbf656afbcb47fe72d947a6"}, + {file = "isort-5.13.2.tar.gz", hash = "sha256:48fdfcb9face5d58a4f6dde2e72a1fb8dcaf8ab26f95ab49fab84c2ddefb0109"}, +] + +[package.extras] +colors = ["colorama (>=0.4.6)"] + [[package]] name = "macrostrat-app-frame" version = "1.2.0" @@ -1045,4 +1059,4 @@ test = ["websockets"] [metadata] lock-version = "2.0" python-versions = "^3.10" -content-hash = "17f3fcb2377005f1e8314292b149f6fe18b37fbc5510457b5c6a4efd2cc4ed59" +content-hash = "c36cfc9026309d101e9b0a36157dee40845a3c102ed0b462594c2608a7a124c4" diff --git a/pyproject.toml b/pyproject.toml index b40a17c..ad7087f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -22,9 +22,14 @@ python-dotenv = "^1.0.0" requests = "^2.27.1" rich = "^13" toml = "^0.10.2" +isort = "^5.13.2" [tool.pytest.ini_options] addopts = "--confcutdir=." [tool.poetry.scripts] -test-app = "test_app:main" \ No newline at end of file +test-app = "test_app:main" + +[tool.isort] +profile = "black" +known_first_party = "macrostrat" \ No newline at end of file diff --git a/utils/macrostrat/utils/__init__.py b/utils/macrostrat/utils/__init__.py index 7235921..62722db 100644 --- a/utils/macrostrat/utils/__init__.py +++ b/utils/macrostrat/utils/__init__.py @@ -5,11 +5,12 @@ import os from contextlib import contextmanager from pathlib import Path -from .logs import setup_stderr_logs, get_logger + +from .logs import get_logger, setup_stderr_logs from .shell import cmd, split_args -def relative_path(base, *parts)->Path: +def relative_path(base, *parts) -> Path: if not os.path.isdir(str(base)): base = os.path.dirname(base) return Path(os.path.join(base, *parts)) diff --git a/utils/macrostrat/utils/logs.py b/utils/macrostrat/utils/logs.py index 0e850e4..3d6964e 100644 --- a/utils/macrostrat/utils/logs.py +++ b/utils/macrostrat/utils/logs.py @@ -1,6 +1,7 @@ import logging from sys import stderr -from colorlog import StreamHandler, ColoredFormatter + +from colorlog import ColoredFormatter, StreamHandler class SparrowLogFormatter(ColoredFormatter): diff --git a/utils/macrostrat/utils/timer.py b/utils/macrostrat/utils/timer.py index 925f7b7..6764c07 100644 --- a/utils/macrostrat/utils/timer.py +++ b/utils/macrostrat/utils/timer.py @@ -2,12 +2,12 @@ A timer for measuring code performance, particularly in web servers. """ -from pydantic import BaseModel -from typing import List from contextlib import contextmanager from contextvars import ContextVar from time import perf_counter +from typing import List +from pydantic import BaseModel code_timer = ContextVar("code_timer", default=None)