From d4e4354929a998efe335f327be7b7c9db924cd9f Mon Sep 17 00:00:00 2001 From: Matthew Keeler Date: Mon, 12 Aug 2024 14:59:25 -0400 Subject: [PATCH] test: Atomically update file data source file --- .github/workflows/ci.yml | 141 ++++++++++-------- Makefile | 2 +- .../integrations/files/file_data_source.py | 19 +++ ldclient/testing/test_file_data_source.py | 42 +++++- 4 files changed, 136 insertions(+), 68 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0aa13560..746fafa9 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -17,7 +17,8 @@ jobs: strategy: fail-fast: false matrix: - python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"] + # python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"] + python-version: ["3.8"] services: redis: @@ -49,6 +50,28 @@ jobs: - name: Run tests run: make test + - run: make test + - run: make test + - run: make test + - run: make test + - run: make test + - run: make test + - run: make test + - run: make test + - run: make test + - run: make test + - run: make test + - run: make test + - run: make test + - run: make test + - run: make test + - run: make test + - run: make test + - run: make test + - run: make test + - run: make test + - run: make test + - name: Verify typehints run: make lint @@ -67,61 +90,61 @@ jobs: test_service_port: 9000 token: ${{ secrets.GITHUB_TOKEN }} - windows: - runs-on: windows-latest - - defaults: - run: - shell: powershell - - strategy: - fail-fast: false - matrix: - python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"] - - steps: - - uses: actions/checkout@v4 - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v5 - with: - python-version: ${{ matrix.python-version }} - - - name: Setup DynamoDB - run: | - $ProgressPreference = "SilentlyContinue" - iwr -outf dynamo.zip https://s3-us-west-2.amazonaws.com/dynamodb-local/dynamodb_local_latest.zip - mkdir dynamo - Expand-Archive -Path dynamo.zip -DestinationPath dynamo - cd dynamo - cmd /c "START /b java -Djava.library.path=./DynamoDBLocal_lib -jar ./DynamoDBLocal.jar" - - - name: Setup Consul - run: | - $ProgressPreference = "SilentlyContinue" - iwr -outf consul.zip https://releases.hashicorp.com/consul/1.4.2/consul_1.4.2_windows_amd64.zip - mkdir consul - Expand-Archive -Path consul.zip -DestinationPath consul - cd consul - sc.exe create "Consul" binPath="$(Get-Location)/consul.exe agent -dev" - sc.exe start "Consul" - - - name: Setup Redis - run: | - $ProgressPreference = "SilentlyContinue" - iwr -outf redis.zip https://github.com/MicrosoftArchive/redis/releases/download/win-3.0.504/Redis-x64-3.0.504.zip - mkdir redis - Expand-Archive -Path redis.zip -DestinationPath redis - cd redis - ./redis-server --service-install - ./redis-server --service-start - Start-Sleep -s 5 - ./redis-cli ping - - - name: Install poetry - uses: abatilo/actions-poetry@7b6d33e44b4f08d7021a1dee3c044e9c253d6439 - - - name: Install requirements - run: poetry install --all-extras - - - name: Run tests - run: make test + # windows: + # runs-on: windows-latest + # + # defaults: + # run: + # shell: powershell + # + # strategy: + # fail-fast: false + # matrix: + # python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"] + # + # steps: + # - uses: actions/checkout@v4 + # - name: Set up Python ${{ matrix.python-version }} + # uses: actions/setup-python@v5 + # with: + # python-version: ${{ matrix.python-version }} + # + # - name: Setup DynamoDB + # run: | + # $ProgressPreference = "SilentlyContinue" + # iwr -outf dynamo.zip https://s3-us-west-2.amazonaws.com/dynamodb-local/dynamodb_local_latest.zip + # mkdir dynamo + # Expand-Archive -Path dynamo.zip -DestinationPath dynamo + # cd dynamo + # cmd /c "START /b java -Djava.library.path=./DynamoDBLocal_lib -jar ./DynamoDBLocal.jar" + # + # - name: Setup Consul + # run: | + # $ProgressPreference = "SilentlyContinue" + # iwr -outf consul.zip https://releases.hashicorp.com/consul/1.4.2/consul_1.4.2_windows_amd64.zip + # mkdir consul + # Expand-Archive -Path consul.zip -DestinationPath consul + # cd consul + # sc.exe create "Consul" binPath="$(Get-Location)/consul.exe agent -dev" + # sc.exe start "Consul" + # + # - name: Setup Redis + # run: | + # $ProgressPreference = "SilentlyContinue" + # iwr -outf redis.zip https://github.com/MicrosoftArchive/redis/releases/download/win-3.0.504/Redis-x64-3.0.504.zip + # mkdir redis + # Expand-Archive -Path redis.zip -DestinationPath redis + # cd redis + # ./redis-server --service-install + # ./redis-server --service-start + # Start-Sleep -s 5 + # ./redis-cli ping + # + # - name: Install poetry + # uses: abatilo/actions-poetry@7b6d33e44b4f08d7021a1dee3c044e9c253d6439 + # + # - name: Install requirements + # run: poetry install --all-extras + # + # - name: Run tests + # run: make test diff --git a/Makefile b/Makefile index fc211437..1ec6bfb2 100644 --- a/Makefile +++ b/Makefile @@ -28,7 +28,7 @@ install: .PHONY: test test: #! Run unit tests test: install - @poetry run pytest $(PYTEST_FLAGS) + @poetry run pytest $(PYTEST_FLAGS) ldclient/testing/test_file_data_source.py .PHONY: lint lint: #! Run type analysis and linting checks diff --git a/ldclient/impl/integrations/files/file_data_source.py b/ldclient/impl/integrations/files/file_data_source.py index d02d5b28..3f184270 100644 --- a/ldclient/impl/integrations/files/file_data_source.py +++ b/ldclient/impl/integrations/files/file_data_source.py @@ -84,6 +84,9 @@ def _load_all(self): for path in self._paths: try: self._load_file(path, all_data) + except FileDataSourceEmpty: + log.warning('No flag data found in any file, continuing with current state') + return except Exception as e: log.error('Unable to load flag data from "%s": %s' % (path, repr(e))) traceback.print_exc() @@ -112,6 +115,10 @@ def _load_file(self, path, all_data): with open(path, 'r') as f: content = f.read() parsed = self._parse_content(content) + + if parsed is None: + raise FileDataSourceEmpty() + for key, flag in parsed.get('flags', {}).items(): _sanitize_json_item(flag) self._add_item(all_data, FEATURES, flag) @@ -165,6 +172,13 @@ def __init__(self, resolved_paths, reloader): class LDWatchdogHandler(watchdog.events.FileSystemEventHandler): def on_any_event(self, event): + if isinstance(event, watchdog.events.FileDeletedEvent): + return + + if isinstance(event, watchdog.events.FileMovedEvent) and event.dest_path in watched_files: + reloader() + return + if event.src_path in watched_files: reloader() @@ -213,4 +227,9 @@ def _check_file_times(self): ret[path] = os.path.getmtime(path) except: ret[path] = None + return ret + + +class FileDataSourceEmpty(Exception): + pass diff --git a/ldclient/testing/test_file_data_source.py b/ldclient/testing/test_file_data_source.py index 56da7762..42212380 100644 --- a/ldclient/testing/test_file_data_source.py +++ b/ldclient/testing/test_file_data_source.py @@ -2,6 +2,8 @@ import os from typing import List +from ldclient.impl.util import log + import pytest import tempfile import threading @@ -116,10 +118,14 @@ def make_temp_file(content): os.close(f) return path -def replace_file(path, content): +def update_file(path, content): with open(path, 'w') as f: f.write(content) +def replace_file(path, content): + new_file = make_temp_file(content) + os.replace(new_file, path) + def test_does_not_load_data_prior_to_start(): path = make_temp_file('{"flagValues":{"key":"value"}}') try: @@ -221,7 +227,7 @@ def test_does_not_allow_duplicate_keys(): os.remove(path1) os.remove(path2) -def test_does_not_reload_modified_file_if_auto_update_is_off(): +def test_does_not_reload_modified_file_if_auto_update_is_off_when_replacing_file(): path = make_temp_file(flag_only_json) try: source = make_data_source(Config("SDK_KEY"), paths = path) @@ -234,15 +240,29 @@ def test_does_not_reload_modified_file_if_auto_update_is_off(): finally: os.remove(path) -def do_auto_update_test(options): +def test_does_not_reload_modified_file_if_auto_update_is_off_when_updating_file(): path = make_temp_file(flag_only_json) + try: + source = make_data_source(Config("SDK_KEY"), paths = path) + source.start() + assert len(store.all(SEGMENTS, lambda x: x)) == 0 + time.sleep(0.5) + update_file(path, segment_only_json) + time.sleep(0.5) + assert len(store.all(SEGMENTS, lambda x: x)) == 0 + finally: + os.remove(path) + +def do_auto_update_test(options, update_fn): + path = make_temp_file(flag_only_json) + options['paths'] = path try: source = make_data_source(Config("SDK_KEY"), **options) source.start() assert len(store.all(SEGMENTS, lambda x: x)) == 0 time.sleep(0.5) - replace_file(path, segment_only_json) + update_fn(path, segment_only_json) deadline = time.time() + 20 while time.time() < deadline: time.sleep(0.1) @@ -252,11 +272,17 @@ def do_auto_update_test(options): finally: os.remove(path) -def test_reloads_modified_file_if_auto_update_is_on(): - do_auto_update_test({ 'auto_update': True }) +def test_reloads_modified_file_if_auto_update_is_on_when_replacing_file(): + do_auto_update_test({ 'auto_update': True }, replace_file) + +def test_reloads_modified_file_in_polling_mode_when_replacing_file(): + do_auto_update_test({ 'auto_update': True, 'force_polling': True, 'poll_interval': 0.1 }, replace_file) + +def test_reloads_modified_file_if_auto_update_is_on_when_updating_file(): + do_auto_update_test({ 'auto_update': True }, update_file) -def test_reloads_modified_file_in_polling_mode(): - do_auto_update_test({ 'auto_update': True, 'force_polling': True, 'poll_interval': 0.1 }) +def test_reloads_modified_file_in_polling_mode_when_updating_file(): + do_auto_update_test({ 'auto_update': True, 'force_polling': True, 'poll_interval': 0.1 }, update_file) def test_evaluates_full_flag_with_client_as_expected(): path = make_temp_file(all_properties_json)