From e2db1ed7092d4389fcd2f0899fd73501cc2de107 Mon Sep 17 00:00:00 2001 From: Donald Campbell <125581724+donaldcampbelljr@users.noreply.github.com> Date: Thu, 30 May 2024 15:12:59 -0400 Subject: [PATCH 1/3] add sqlite option if included in config file, #192 --- pipestat/pipestat.py | 6 +++++- tests/conftest.py | 10 ++++++++++ tests/data/config_sqllite.yaml | 5 +++++ tests/data/sample_output_schema_sqlite.yaml | 14 ++++++++++++++ tests/test_db_only_mode.py | 18 ++++++++++++++++++ 5 files changed, 52 insertions(+), 1 deletion(-) create mode 100644 tests/data/config_sqllite.yaml create mode 100644 tests/data/sample_output_schema_sqlite.yaml diff --git a/pipestat/pipestat.py b/pipestat/pipestat.py index 25ac6ac4..1ce173d2 100644 --- a/pipestat/pipestat.py +++ b/pipestat/pipestat.py @@ -400,7 +400,11 @@ def initialize_dbbackend(self, record_identifier, show_db_logs): dbconf = self.cfg[CONFIG_KEY].exp[ CFG_DATABASE_KEY ] # the .exp expands the paths before url construction - self.cfg[DB_URL] = construct_db_url(dbconf) + if "sqlite_url" in dbconf: + sqlite_url = f"sqlite:///{dbconf['sqlite_url']}" + self.cfg[DB_URL] = sqlite_url + else: + self.cfg[DB_URL] = construct_db_url(dbconf) except KeyError: raise PipestatDatabaseError(f"No database section ('{CFG_DATABASE_KEY}') in config") self._show_db_logs = show_db_logs diff --git a/tests/conftest.py b/tests/conftest.py index f4a634c1..41275419 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -79,6 +79,11 @@ def schema_file_path(): return get_data_file_path("sample_output_schema.yaml") +@pytest.fixture +def schema_file_path_sqlite(): + return get_data_file_path("sample_output_schema_sqlite.yaml") + + @pytest.fixture def highlight_schema_file_path(): return get_data_file_path("sample_output_schema_highlight.yaml") @@ -94,6 +99,11 @@ def config_file_path(): return get_data_file_path("config.yaml") +@pytest.fixture +def config_file_path_sqllite(): + return get_data_file_path("config_sqllite.yaml") + + @pytest.fixture def config_no_schema_file_path(): return get_data_file_path("config_no_schema.yaml") diff --git a/tests/data/config_sqllite.yaml b/tests/data/config_sqllite.yaml new file mode 100644 index 00000000..56b38591 --- /dev/null +++ b/tests/data/config_sqllite.yaml @@ -0,0 +1,5 @@ +project_name: test +record_identifier: sample1 +schema_path: sample_output_schema_sqlite.yaml +database: + sqlite_url: /home/drc/sqlite_databases/pipestattest diff --git a/tests/data/sample_output_schema_sqlite.yaml b/tests/data/sample_output_schema_sqlite.yaml new file mode 100644 index 00000000..d3687b72 --- /dev/null +++ b/tests/data/sample_output_schema_sqlite.yaml @@ -0,0 +1,14 @@ +title: An example Pipestat output schema +description: A pipeline that uses pipestat to report sample and project level results. +type: object +properties: + pipeline_name: "default_pipeline_name" + samples: + type: object + properties: + number_of_things: + type: integer + description: "Number of things" + percentage_of_things: + type: number + description: "Percentage of things" \ No newline at end of file diff --git a/tests/test_db_only_mode.py b/tests/test_db_only_mode.py index 28f316cf..828fdfa5 100644 --- a/tests/test_db_only_mode.py +++ b/tests/test_db_only_mode.py @@ -134,3 +134,21 @@ def test_select_pagination( result = psm.select_records(cursor=offset, limit=limit) print(result) assert len(result["records"]) == min(max((psm.record_count - offset), 0), limit) + + +@pytest.mark.skip(reason="only works for local testing") +class TestSQLLITE: + def test_manager_can_be_built_without_exception( + self, config_file_path_sqllite, schema_file_path_sqlite + ): + # with ContextManagerDBTesting(DB_URL): + try: + SamplePipestatManager( + schema_path=schema_file_path_sqlite, + record_identifier="irrelevant", + database_only=True, + config_file=config_file_path_sqllite, + ) + print("done") + except Exception as e: + pytest.fail(f"Pipestat manager construction failed: {e})") From 57bcda07f79ec076482d495dfac8028b23fb0c1d Mon Sep 17 00:00:00 2001 From: Donald Campbell <125581724+donaldcampbelljr@users.noreply.github.com> Date: Thu, 18 Jul 2024 11:22:06 -0400 Subject: [PATCH 2/3] make it such that the sqllite test can run anywhere --- tests/data/config_sqllite.yaml | 5 ----- tests/test_db_only_mode.py | 38 ++++++++++++++++++++-------------- 2 files changed, 23 insertions(+), 20 deletions(-) delete mode 100644 tests/data/config_sqllite.yaml diff --git a/tests/data/config_sqllite.yaml b/tests/data/config_sqllite.yaml deleted file mode 100644 index 56b38591..00000000 --- a/tests/data/config_sqllite.yaml +++ /dev/null @@ -1,5 +0,0 @@ -project_name: test -record_identifier: sample1 -schema_path: sample_output_schema_sqlite.yaml -database: - sqlite_url: /home/drc/sqlite_databases/pipestattest diff --git a/tests/test_db_only_mode.py b/tests/test_db_only_mode.py index 828fdfa5..aac3f238 100644 --- a/tests/test_db_only_mode.py +++ b/tests/test_db_only_mode.py @@ -1,3 +1,5 @@ +from tempfile import NamedTemporaryFile + import pytest from pipestat import SamplePipestatManager @@ -136,19 +138,25 @@ def test_select_pagination( assert len(result["records"]) == min(max((psm.record_count - offset), 0), limit) -@pytest.mark.skip(reason="only works for local testing") class TestSQLLITE: - def test_manager_can_be_built_without_exception( - self, config_file_path_sqllite, schema_file_path_sqlite - ): - # with ContextManagerDBTesting(DB_URL): - try: - SamplePipestatManager( - schema_path=schema_file_path_sqlite, - record_identifier="irrelevant", - database_only=True, - config_file=config_file_path_sqllite, - ) - print("done") - except Exception as e: - pytest.fail(f"Pipestat manager construction failed: {e})") + def test_manager_can_be_built_without_exception(self, schema_file_path_sqlite): + + with NamedTemporaryFile() as f: + sqllite_url = f"sqlite:///{f.name}" + config_dict = { + "project_name": "test", + "record_identifier": "sample1", + "schema_path": "sample_output_schema_sqlite.yaml", + "database": {"sqlite_url": f.name}, + } + + with ContextManagerDBTesting(sqllite_url): + try: + SamplePipestatManager( + schema_path=schema_file_path_sqlite, + record_identifier="irrelevant", + database_only=True, + config_dict=config_dict, + ) + except Exception as e: + pytest.fail(f"Pipestat manager construction failed: {e})") From f75adcabe009e07e068327304ab3e460770ba2ce Mon Sep 17 00:00:00 2001 From: Donald Campbell <125581724+donaldcampbelljr@users.noreply.github.com> Date: Thu, 18 Jul 2024 11:23:19 -0400 Subject: [PATCH 3/3] remove unnecessary test fixture --- tests/conftest.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index 41275419..99974c27 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -99,11 +99,6 @@ def config_file_path(): return get_data_file_path("config.yaml") -@pytest.fixture -def config_file_path_sqllite(): - return get_data_file_path("config_sqllite.yaml") - - @pytest.fixture def config_no_schema_file_path(): return get_data_file_path("config_no_schema.yaml")