Remove management command, move to new dispatcher method, fix job log…

…ging and signature
nautobot · Sep 10, 2023 · 8773faf · 8773faf
1 parent ba064e4
commit 8773faf
Show file tree

Hide file tree

Showing 35 changed files with 841 additions and 5,336 deletions.
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -79,8 +79,8 @@ jobs:
     strategy:
       fail-fast: true
       matrix:
-        python-version: ["3.10"]  # ["3.8"]
-        nautobot-version: ["2.0.0-rc.2"]  # ["1.5.3"]
+        python-version: ["3.11"]
+        nautobot-version: ["2.0.0-rc.2"]
     env:
       INVOKE_NAUTOBOT_GOLDEN_CONFIG_PYTHON_VER: "${{ matrix.python-version }}"
       INVOKE_NAUTOBOT_GOLDEN_CONFIG_NAUTOBOT_VER: "${{ matrix.nautobot-version }}"
@@ -116,7 +116,7 @@ jobs:
     strategy:
       fail-fast: true
       matrix:
-        python-version: ["3.10"]  # ["3.7", "3.8", "3.9", "3.10"]
+        python-version: ["3.8", "3.9", "3.10", "3.11"]
         db-backend: ["postgresql"]
         nautobot-version: ["2.0.0-rc.2"]  # ["latest"]
         # The include is a method to limit the amount of jobs ran. This essentially
@@ -125,13 +125,13 @@ jobs:
         include:
           - python-version: "3.11"
             db-backend: "postgresql"
-            nautobot-version: "2.0.0-rc.2"  # "1.5.3"
-          - python-version: "3.10"  # "3.7"
+            nautobot-version: "2.0.0-rc.2"
+          - python-version: "3.11"
+            db-backend: "mysql"
+            nautobot-version: "2.0.0-rc.2"
+          - python-version: "3.8"
             db-backend: "mysql"
-            nautobot-version: "2.0.0-rc.2" # "1.5.3"
-          #- python-version: "3.10"
-          #  db-backend: "mysql"
-          #  nautobot-version: "latest"
+            nautobot-version: "latest"
     runs-on: "ubuntu-20.04"
     env:
       INVOKE_NAUTOBOT_GOLDEN_CONFIG_PYTHON_VER: "${{ matrix.python-version }}"
@@ -206,7 +206,7 @@ jobs:
       - name: "Set up Python"
         uses: "actions/setup-python@v2"
         with:
-          python-version: "3.10"  # "3.9"
+          python-version: "3.11"
       - name: "Install Python Packages"
         run: "pip install poetry"
       - name: "Set env"

diff --git a/.readthedocs.yaml b/.readthedocs.yaml
@@ -10,7 +10,7 @@ version: 2
 build:
   os: "ubuntu-22.04"
   tools:
-    python: "3.10"
+    python: "3.11"
 
 mkdocs:
   configuration: "mkdocs.yml"

diff --git a/development/Dockerfile b/development/Dockerfile
@@ -6,11 +6,11 @@
 # -------------------------------------------------------------------------------------
 # !!! USE CAUTION WHEN MODIFYING LINES BELOW
 
-# Accepts a desired Nautobot version as build argument, default to 1.5.3
+# Accepts a desired Nautobot version as build argument, default to 2.0.0
 ARG NAUTOBOT_VER="2.0.0-rc.2"
 
-# Accepts a desired Python version as build argument, default to 3.7
-ARG PYTHON_VER="3.10"
+# Accepts a desired Python version as build argument, default to 3.11
+ARG PYTHON_VER="3.11"
 
 # Retrieve published development image of Nautobot base which should include most CI dependencies
 FROM ghcr.io/nautobot/nautobot-dev:${NAUTOBOT_VER}-py${PYTHON_VER}

diff --git a/development/docker-compose.dev.yml b/development/docker-compose.dev.yml
@@ -12,6 +12,13 @@ services:
     volumes:
       - "./nautobot_config.py:/opt/nautobot/nautobot_config.py"
       - "../:/source"
+      # Helper method to mount on top of the python implementations, assuming you are using py3.11 and
+      # have all of your projects in the same directory. Uncomment out as required.
+      # - "../../netutils/netutils:/opt/nautobot/lib/python3.11/site-packages/netutils"
+      # - "../../nornir-nautobot/nornir_nautobot:/opt/nautobot/lib/python3.11/site-packages/nornir_nautobot"
+      # - "../../nautobot-plugin-nornir/nautobot_plugin_nornir:/opt/nautobot/lib/python3.11/site-packages/nautobot_plugin_nornir"
+      # - "../../nautobot/nautobot:/opt/nautobot/lib/python3.11/site-packages/nautobot"
+
   docs:
     entrypoint: "mkdocs serve -v -a 0.0.0.0:8080"
     ports:
@@ -26,6 +33,12 @@ services:
     volumes:
       - "./nautobot_config.py:/opt/nautobot/nautobot_config.py"
       - "../:/source"
+      # Helper method to mount on top of the python implementations, assuming you are using py3.11 and
+      # have all of your projects in the same directory. Uncomment out as required.
+      # - "../../netutils/netutils:/opt/nautobot/lib/python3.11/site-packages/netutils"
+      # - "../../nornir-nautobot/nornir_nautobot:/opt/nautobot/lib/python3.11/site-packages/nornir_nautobot"
+      # - "../../nautobot-plugin-nornir/nautobot_plugin_nornir:/opt/nautobot/lib/python3.11/site-packages/nautobot_plugin_nornir"
+      # - "../../nautobot/nautobot:/opt/nautobot/lib/python3.11/site-packages/nautobot"
 # To expose postgres or redis to the host uncomment the following
 # postgres:
 #   ports:

diff --git a/docs/admin/release_notes/version_1.4.md b/docs/admin/release_notes/version_1.4.md
@@ -6,55 +6,55 @@
 
 ### Changed
 
-- [519](https://github.com/nautobot/nautobot-plugin-golden-config/pull/519) - docs-only: large fixes and template troubleshooting section.
+- [#519](https://github.com/nautobot/nautobot-plugin-golden-config/pull/519) - docs-only: large fixes and template troubleshooting section.
 
 ### Fixed
 
-- [492](https://github.com/nautobot/nautobot-plugin-golden-config/pull/492) - Fix count of in scope devices on settings detail view.
-- [498](https://github.com/nautobot/nautobot-plugin-golden-config/pull/498) - Fix deepdiff dependency.
-- [501](https://github.com/nautobot/nautobot-plugin-golden-config/pull/501) - Update docs for adding CustomField data with datasources.
-- [503](https://github.com/nautobot/nautobot-plugin-golden-config/pull/503) - Switch from deprecated FilterSet to new FilterSetMixin.
-- [504](https://github.com/nautobot/nautobot-plugin-golden-config/pull/504) - Fix extend queryfilter to export.
-- [511](https://github.com/nautobot/nautobot-plugin-golden-config/pull/511) - Fix `log_failure` function missing argument.
-- [523](https://github.com/nautobot/nautobot-plugin-golden-config/pull/523) - Fix docs site by pinning dev dependencies.
-- [530](https://github.com/nautobot/nautobot-plugin-golden-config/pull/530) - Fix, removing ConfigCompliance model import from 0005 migration.
+- [#492](https://github.com/nautobot/nautobot-plugin-golden-config/pull/492) - Fix count of in scope devices on settings detail view.
+- [#498](https://github.com/nautobot/nautobot-plugin-golden-config/pull/498) - Fix deepdiff dependency.
+- [#501](https://github.com/nautobot/nautobot-plugin-golden-config/pull/501) - Update docs for adding CustomField data with datasources.
+- [#503](https://github.com/nautobot/nautobot-plugin-golden-config/pull/503) - Switch from deprecated FilterSet to new FilterSetMixin.
+- [#504](https://github.com/nautobot/nautobot-plugin-golden-config/pull/504) - Fix extend queryfilter to export.
+- [#511](https://github.com/nautobot/nautobot-plugin-golden-config/pull/511) - Fix `log_failure` function missing argument.
+- [#523](https://github.com/nautobot/nautobot-plugin-golden-config/pull/523) - Fix docs site by pinning dev dependencies.
+- [#530](https://github.com/nautobot/nautobot-plugin-golden-config/pull/530) - Fix, removing ConfigCompliance model import from 0005 migration.
 
 ## v1.4.1 - 2023-05
 
 ### Fixed
 
-- [488](https://github.com/nautobot/nautobot-plugin-golden-config/pull/488) - Fix Golden Config Settings Buttons.
+- [#488](https://github.com/nautobot/nautobot-plugin-golden-config/pull/488) - Fix Golden Config Settings Buttons.
 
 ## v1.4.0 - 2023-05
 
 ### Added
 
-- [445](https://github.com/nautobot/nautobot-plugin-golden-config/pull/445) - Add validation for Settings sot_agg_query.
-- [449](https://github.com/nautobot/nautobot-plugin-golden-config/pull/449) - Allows for custom kwargs to `get_secret_by_secret_group_slug`.
-- [470](https://github.com/nautobot/nautobot-plugin-golden-config/pull/470) - Enhance UI settings detail object view.
-- [473](https://github.com/nautobot/nautobot-plugin-golden-config/pull/473) - Add status selection field to job filtering.
-- [480](https://github.com/nautobot/nautobot-plugin-golden-config/pull/480) - Add compliance summary to default tenant view.
+- [#445](https://github.com/nautobot/nautobot-plugin-golden-config/pull/445) - Add validation for Settings sot_agg_query.
+- [#449](https://github.com/nautobot/nautobot-plugin-golden-config/pull/449) - Allows for custom kwargs to `get_secret_by_secret_group_slug`.
+- [#470](https://github.com/nautobot/nautobot-plugin-golden-config/pull/470) - Enhance UI settings detail object view.
+- [#473](https://github.com/nautobot/nautobot-plugin-golden-config/pull/473) - Add status selection field to job filtering.
+- [#480](https://github.com/nautobot/nautobot-plugin-golden-config/pull/480) - Add compliance summary to default tenant view.
 
 ### Changed
 
-- [414](https://github.com/nautobot/nautobot-plugin-golden-config/pull/414) - Update application description for UI.
-- [407](https://github.com/nautobot/nautobot-plugin-golden-config/pull/407) - Update branching policy in contributing docs.
-- [417](https://github.com/nautobot/nautobot-plugin-golden-config/pull/417) - Changed extends base.html to extends generic/object_detail.html.
-- [434](https://github.com/nautobot/nautobot-plugin-golden-config/pull/434) - Upgrade deepdiff dependency to 6.2.0.
-- [451](https://github.com/nautobot/nautobot-plugin-golden-config/pull/451) - Tune Dependabot.
-- [459](https://github.com/nautobot/nautobot-plugin-golden-config/pull/459) - Update tasks.py to meet current standards.
-- [464](https://github.com/nautobot/nautobot-plugin-golden-config/pull/464) - Update ordering on compliance views.
-- [471](https://github.com/nautobot/nautobot-plugin-golden-config/pull/471) - Migrate to using NautobotUIViewset and other initial 2.x prep work.
-- [481](https://github.com/nautobot/nautobot-plugin-golden-config/pull/481) - Update filtersets for rack-group to extend proper TreeNode parent.
+- [#414](https://github.com/nautobot/nautobot-plugin-golden-config/pull/414) - Update application description for UI.
+- [#407](https://github.com/nautobot/nautobot-plugin-golden-config/pull/407) - Update branching policy in contributing docs.
+- [#417](https://github.com/nautobot/nautobot-plugin-golden-config/pull/417) - Changed extends base.html to extends generic/object_detail.html.
+- [#434](https://github.com/nautobot/nautobot-plugin-golden-config/pull/434) - Upgrade deepdiff dependency to 6.2.0.
+- [#451](https://github.com/nautobot/nautobot-plugin-golden-config/pull/451) - Tune Dependabot.
+- [#459](https://github.com/nautobot/nautobot-plugin-golden-config/pull/459) - Update tasks.py to meet current standards.
+- [#464](https://github.com/nautobot/nautobot-plugin-golden-config/pull/464) - Update ordering on compliance views.
+- [#471](https://github.com/nautobot/nautobot-plugin-golden-config/pull/471) - Migrate to using NautobotUIViewset and other initial 2.x prep work.
+- [#481](https://github.com/nautobot/nautobot-plugin-golden-config/pull/481) - Update filtersets for rack-group to extend proper TreeNode parent.
 
 ### Fixed
 
-- [436](https://github.com/nautobot/nautobot-plugin-golden-config/pull/436) - Update FAQ for how compliance works.
-- [444](https://github.com/nautobot/nautobot-plugin-golden-config/pull/444) - `app_faq.md` references incorrect `Cisco IOS XR` platform slug.
-- [446](https://github.com/nautobot/nautobot-plugin-golden-config/pull/446) - Fix mysql not working in github actions.
-- [450](https://github.com/nautobot/nautobot-plugin-golden-config/pull/450) - Make ConfigReplace export match import.
-- [456](https://github.com/nautobot/nautobot-plugin-golden-config/pull/456) - Fix postprocessing to use Sandbox Jinja2 environment.
-- [461](https://github.com/nautobot/nautobot-plugin-golden-config/pull/461) - Moves dependabot config to proper location.
-- [463](https://github.com/nautobot/nautobot-plugin-golden-config/pull/463) - Fix Json render in compliance reporting template.
-- [468](https://github.com/nautobot/nautobot-plugin-golden-config/pull/468) - Fix GoldenConfig list view and csv export.
-- [474](https://github.com/nautobot/nautobot-plugin-golden-config/pull/474) - Docs update: Fix multiple typos.
+- [#436](https://github.com/nautobot/nautobot-plugin-golden-config/pull/436) - Update FAQ for how compliance works.
+- [#444](https://github.com/nautobot/nautobot-plugin-golden-config/pull/444) - `app_faq.md` references incorrect `Cisco IOS XR` platform slug.
+- [#446](https://github.com/nautobot/nautobot-plugin-golden-config/pull/446) - Fix mysql not working in github actions.
+- [#450](https://github.com/nautobot/nautobot-plugin-golden-config/pull/450) - Make ConfigReplace export match import.
+- [#456](https://github.com/nautobot/nautobot-plugin-golden-config/pull/456) - Fix postprocessing to use Sandbox Jinja2 environment.
+- [#461](https://github.com/nautobot/nautobot-plugin-golden-config/pull/461) - Moves dependabot config to proper location.
+- [#463](https://github.com/nautobot/nautobot-plugin-golden-config/pull/463) - Fix Json render in compliance reporting template.
+- [#468](https://github.com/nautobot/nautobot-plugin-golden-config/pull/468) - Fix GoldenConfig list view and csv export.
+- [#474](https://github.com/nautobot/nautobot-plugin-golden-config/pull/474) - Docs update: Fix multiple typos.
diff --git a/docs/admin/release_notes/version_1.5.md b/docs/admin/release_notes/version_1.5.md
@@ -11,17 +11,17 @@
 
 ### Added
 
-- [455](https://github.com/nautobot/nautobot-plugin-golden-config/pull/455) - Add metrics for Golden Config plugin.
-- [485](https://github.com/nautobot/nautobot-plugin-golden-config/pull/485) - Custom compliance for CLI and JSON rules.
-- [487](https://github.com/nautobot/nautobot-plugin-golden-config/pull/487) - Implement native JSON support.
-- [527](https://github.com/nautobot/nautobot-plugin-golden-config/pull/527) - Add the ability to update Jinja environment setting from nautobot_config.
-- [558](https://github.com/nautobot/nautobot-plugin-golden-config/pull/558) - Updated Filters for various models, including adding an experimental `_isnull` on DateTime objects.
+- [#455](https://github.com/nautobot/nautobot-plugin-golden-config/pull/455) - Add metrics for Golden Config plugin.
+- [#485](https://github.com/nautobot/nautobot-plugin-golden-config/pull/485) - Custom compliance for CLI and JSON rules.
+- [#487](https://github.com/nautobot/nautobot-plugin-golden-config/pull/487) - Implement native JSON support.
+- [#527](https://github.com/nautobot/nautobot-plugin-golden-config/pull/527) - Add the ability to update Jinja environment setting from nautobot_config.
+- [#558](https://github.com/nautobot/nautobot-plugin-golden-config/pull/558) - Updated Filters for various models, including adding an experimental `_isnull` on DateTime objects.
 
 ### Changed
 
-- [485](https://github.com/nautobot/nautobot-plugin-golden-config/pull/485) - Changed the behavior of custom compliance to a boolean vs toggle between cli, json, and custom.
+- [#485](https://github.com/nautobot/nautobot-plugin-golden-config/pull/485) - Changed the behavior of custom compliance to a boolean vs toggle between cli, json, and custom.
 
 ### Fixed
 
-- [505](https://github.com/nautobot/nautobot-plugin-golden-config/pull/505) - fixes imports and choice definitions in the compliance nornir play.
-- [513](https://github.com/nautobot/nautobot-plugin-golden-config/pull/513) - Fixed issue with native JSON support with `get_config_element` function.
+- [#505](https://github.com/nautobot/nautobot-plugin-golden-config/pull/505) - fixes imports and choice definitions in the compliance nornir play.
+- [#513](https://github.com/nautobot/nautobot-plugin-golden-config/pull/513) - Fixed issue with native JSON support with `get_config_element` function.
diff --git a/docs/dev/dev_environment.md b/docs/dev/dev_environment.md
@@ -15,7 +15,7 @@ The [Invoke](http://www.pyinvoke.org/) library is used to provide some helper co
 
 - `nautobot_ver`: the version of Nautobot to use as a base for any built docker containers (default: latest)
 - `project_name`: the default docker compose project name (default: `nautobot_golden_config`)
-- `python_ver`: the version of Python to use as a base for any built docker containers (default: 3.8)
+- `python_ver`: the version of Python to use as a base for any built docker containers (default: 3.11)
 - `local`: a boolean flag indicating if invoke tasks should be run on the host or inside the docker containers (default: False, commands will be run in docker containers)
 - `compose_dir`: the full path to a directory containing the project compose files
 - `compose_files`: a list of compose files applied in order (see [Multiple Compose files](https://docs.docker.com/compose/extends/#multiple-compose-files) for more information)
@@ -187,7 +187,7 @@ The first thing you need to do is build the necessary Docker image for Nautobot
 #14 exporting layers
 #14 exporting layers 1.2s done
 #14 writing image sha256:2d524bc1665327faa0d34001b0a9d2ccf450612bf8feeb969312e96a2d3e3503 done
-#14 naming to docker.io/nautobot-golden-config/nautobot:latest-py3.7 done
+#14 naming to docker.io/nautobot-golden-config/nautobot:latest-py3.11 done
 ```
 
 ### Invoke - Starting the Development Environment
@@ -218,9 +218,9 @@ This will start all of the Docker containers used for hosting Nautobot. You shou
 ```bash
 ➜ docker ps
 ****CONTAINER ID   IMAGE                            COMMAND                  CREATED          STATUS          PORTS                                       NAMES
-ee90fbfabd77   nautobot-golden-config/nautobot:latest-py3.7   "nautobot-server rqw…"   16 seconds ago   Up 13 seconds                                               nautobot_golden_config_worker_1
-b8adb781d013   nautobot-golden-config/nautobot:latest-py3.7   "/docker-entrypoint.…"   20 seconds ago   Up 15 seconds   0.0.0.0:8080->8080/tcp, :::8080->8080/tcp   nautobot_golden_config_nautobot_1
-d64ebd60675d   nautobot-golden-config/nautobot:latest-py3.7   "mkdocs serve -v -a …"   25 seconds ago   Up 18 seconds   0.0.0.0:8001->8080/tcp, :::8001->8080/tcp   nautobot_golden_config_docs_1
+ee90fbfabd77   nautobot-golden-config/nautobot:latest-py3.11   "nautobot-server rqw…"   16 seconds ago   Up 13 seconds                                               nautobot_golden_config_worker_1
+b8adb781d013   nautobot-golden-config/nautobot:latest-py3.11   "/docker-entrypoint.…"   20 seconds ago   Up 15 seconds   0.0.0.0:8080->8080/tcp, :::8080->8080/tcp   nautobot_golden_config_nautobot_1
+d64ebd60675d   nautobot-golden-config/nautobot:latest-py3.11   "mkdocs serve -v -a …"   25 seconds ago   Up 18 seconds   0.0.0.0:8001->8080/tcp, :::8001->8080/tcp   nautobot_golden_config_docs_1
 e72d63129b36   postgres:13-alpine               "docker-entrypoint.s…"   25 seconds ago   Up 19 seconds   0.0.0.0:5432->5432/tcp, :::5432->5432/tcp   nautobot_golden_config_postgres_1
 96c6ff66997c   redis:6-alpine                   "docker-entrypoint.s…"   25 seconds ago   Up 21 seconds   0.0.0.0:6379->6379/tcp, :::6379->6379/tcp   nautobot_golden_config_redis_1
 ```
@@ -399,7 +399,7 @@ namespace.configure(
     {
         "nautobot_golden_config": {
             ...
-            "python_ver": "3.7",
+            "python_ver": "3.11",
 	    ...
         }
     }

diff --git a/docs/user/troubleshooting/troubleshoot_dispatchers.md b/docs/user/troubleshooting/troubleshoot_dispatchers.md
@@ -14,6 +14,8 @@ This occurs when a Golden Config job is executed with a Nautobot `platform`, and
 
 How is the dispatcher loaded?
 
+TODO: 2.0: Change to custom_dispatcher
+
 1. Job initializes Nornir and the method is called with `get_dispatcher()` function from Nautobot-Plugin-Nornir.
 2. Nornir initialization looks in the DEFAULT_DISPATCHER map for the platform network_driver from [nornir-nautobot](https://github.com/nautobot/nornir-nautobot/blob/64baa8a24d21d9ec14c32be569e2b51cd0bd1cd1/nornir_nautobot/plugins/tasks/dispatcher/__init__.py#L12) mapping.
 3. Merge this mapping with anything directly configured in Golden Config [dispatcher mapping]().