From cc22db389ac856deeac7f7ed00590a17b10e93aa Mon Sep 17 00:00:00 2001 From: Elizabeth Sall Date: Wed, 9 Oct 2024 17:13:26 -0700 Subject: [PATCH] Replaced flake8 with Ruff Flake8 giving some trouble b/c API is deprecated. Added additional tests for pycode to make sure it worked as expected. --- projectcard/io.py | 4 ++-- projectcard/validate.py | 38 ++++++++++++++++++++++---------------- pyproject.toml | 2 +- requirements.tests.txt | 1 - requirements.txt | 2 +- tests/test_validate.py | 35 +++++++++++++++++++++++++++++++++++ 6 files changed, 61 insertions(+), 21 deletions(-) diff --git a/projectcard/io.py b/projectcard/io.py index 8a57c6e..c6a4a32 100644 --- a/projectcard/io.py +++ b/projectcard/io.py @@ -182,12 +182,12 @@ def _change_keys(obj: dict, convert: Callable = _replace_selected) -> dict: VALID_EXT = list(_read_method_map.keys()) -def read_card(filepath: ProjectCardFilepath, validate: bool = False): +def read_card(filepath: ProjectCardFilepath, validate: bool = True): """Read single project card from a path and return project card object. Args: filepath: file where the project card is. - validate: if True, will validate the project card schemea + validate: if True, will validate the project card schema. Defaults to True. """ if not Path(filepath).is_file(): msg = f"Cannot find project card file: {filepath}" diff --git a/projectcard/validate.py b/projectcard/validate.py index 81984dd..0945b8f 100644 --- a/projectcard/validate.py +++ b/projectcard/validate.py @@ -7,7 +7,6 @@ from typing import Optional, Union import jsonref -from flake8.api import legacy as flake8 from jsonschema import validate from jsonschema.exceptions import SchemaError, ValidationError @@ -17,12 +16,14 @@ ROOTDIR = Path(__file__).resolve().parent PROJECTCARD_SCHEMA = ROOTDIR / "schema" / "projectcard.json" -# Errors to catch in valdiating "wrangler" project cards which use python code. -# E9 Runtime -# F63 undefined name name -# F823 local variable name ... referenced before assignment -# F405 name may be undefined, or defined from star imports: module -FLAKE8_ERRORS = ["E9", "F821", "F823", "F405"] +CRITICAL_ERRORS = ["E9", "F821", "F823", "F405"] +""" +Errors in Ruff that will cause a code execution failure. +E9: Syntax errors. +F821: Undefined name. +F823: Local variable referenced before assignment. +F405: Name may be undefined, or defined from star imports. +""" def _open_json(schema_path: Path) -> dict: @@ -203,7 +204,6 @@ def _validate_pycode(jsondata: dict, mocked_vars: list[str] = DEFAULT_MOCKED_VAR jsondata: project card json data as a python dictionary mocked_vars: list of variables available in the execution of the code """ - style_guide = flake8.get_style_guide(select=FLAKE8_ERRORS, ignore=["E", "F", "W"]) dir = TemporaryDirectory() tmp_py_path = Path(dir.name) / "tempcode.py" CardLogger.debug(f"Storing temporary python files at: {tmp_py_path!s}") @@ -215,14 +215,20 @@ def _validate_pycode(jsondata: dict, mocked_vars: list[str] = DEFAULT_MOCKED_VAR with tmp_py_path.open("w") as py_file: py_file.write(py_file_contents) + import subprocess - report = style_guide.check_files([tmp_py_path]) - - if report.total_errors: + try: + result = subprocess.run( + ["ruff", "check", tmp_py_path, "--select", ",".join(CRITICAL_ERRORS)], + capture_output=True, + text=True, + check=True, + ) + except subprocess.CalledProcessError as e: CardLogger.error(f"Errors found in {jsondata['project']}") CardLogger.debug(f"FILE CONTENTS\n{py_file_contents}") - errors = {c: report.get_statistics(c) for c in FLAKE8_ERRORS if report.get_statistics(c)} - CardLogger.debug(f"Flake 8 Report:\n {errors}") - msg = f"Found {report.total_errors} errors in {jsondata['project']}" - raise PycodeError(msg) - dir.cleanup() + CardLogger.debug(f"Ruff Report:\n {e.stdout}") + msg = f"Found errors in {jsondata['project']}" + raise PycodeError(msg) from e + finally: + dir.cleanup() diff --git a/pyproject.toml b/pyproject.toml index 82b9bc3..1223d55 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -25,12 +25,12 @@ classifiers = [ ] requires-python = ">=3.9" dependencies = [ - "flake8", "jsonref", "jsonschema", "pydantic>=2.0", "pyyaml", "tabulate", + "ruff", "toml", ] diff --git a/requirements.tests.txt b/requirements.tests.txt index 66058fd..716c575 100644 --- a/requirements.tests.txt +++ b/requirements.tests.txt @@ -2,4 +2,3 @@ mypy pre-commit pytest pytest-cov -ruff diff --git a/requirements.txt b/requirements.txt index f0683c0..6c77297 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,6 @@ -flake8 jsonref jsonschema pydantic>=2.0 pyyaml +ruff toml diff --git a/tests/test_validate.py b/tests/test_validate.py index 54d3cfa..4a19c0f 100644 --- a/tests/test_validate.py +++ b/tests/test_validate.py @@ -1,6 +1,11 @@ """Tests for the projectcard.validate module.""" +import pytest + +from projectcard import ProjectCard +from projectcard.errors import ProjectCardValidationError, PycodeError from projectcard.logger import CardLogger +from projectcard.validate import validate_card def test_update_dict_with_schema_defaults(): @@ -134,3 +139,33 @@ def test_update_project_card_with_defaults(): card = ProjectCard(project_card_data) CardLogger.debug(f"card:\n{card}") assert card.transit_service_deletion["clean_shapes"] is False + + +def test_good_pycode_(request): + """Make sure bad pycode syntax will raise an error.""" + CardLogger.info(f"--Starting: {request.node.name}") + + _pycode = "roadway_net.links_df.loc[roadway_net.links_df['lanes'] == 5, 'lanes'] = 12" + project_data = { + "project": "Test Project", + "self_obj_type": "RoadwayNetwork", + "pycode": _pycode, + } + project_card = ProjectCard(project_data) + valid = project_card.validate() + assert valid + + +def test_bad_pycode_(request): + """Make sure bad pycode syntax will raise an error.""" + CardLogger.info(f"--Starting: {request.node.name}") + + _pycode = "roadway_net.links_df.loc[[roadway_net.links_df['lanes'] == 5, 'lanes'] = 12" + project_data = { + "project": "Test Project", + "self_obj_type": "RoadwayNetwork", + "pycode": _pycode, + } + project_card = ProjectCard(project_data) + with pytest.raises(PycodeError): + valid = project_card.validate()