support_202306 (#82)

* peeringdb-py commands to setup and run peeringdb server #1219

* modernize

---------

Co-authored-by: 20C <[email protected]>

.github/workflows/tests.yml

@@ -21,7 +21,7 @@ jobs:
       fail-fast: true
       matrix:
         os: [ "ubuntu-latest", "macos-latest" ]
-        python-version: [ "3.7", "3.8", "3.9", "3.10" ]
+        python-version: [ "3.8", "3.9", "3.10", "3.11" ]
     runs-on: ${{ matrix.os }}
     steps:
       - name: Check out repository

CHANGELOG.md

@@ -2,6 +2,15 @@
 
 
 ## Unreleased
+### Added
+- django 4.2 support
+- python 3.11 support
+- support for fetching data from peeringdb cache servers
+- support for quickly setting a local snapshot of peeringdb server via the `peeringdb server` commands
+### Removed
+- python 3.7 support
+- django 2.2 support
+- django 3.0 support
 
 
 ## 1.5.0

CHANGELOG.yaml

@@ -1,9 +1,16 @@
 Unreleased:
-  added: []
+  added:
+  - django 4.2 support
+  - python 3.11 support
+  - support for fetching data from peeringdb cache servers
+  - support for quickly setting a local snapshot of peeringdb server via the `peeringdb server` commands
   fixed: []
   changed: []
   deprecated: []
-  removed: []
+  removed: 
+  - python 3.7 support
+  - django 2.2 support
+  - django 3.0 support
   security: []
 1.5.0:
   added:

docs/cli.md

@@ -79,3 +79,36 @@ Then add a cron job to keep it in sync, for example, once a day at between midni
 Or, if your cron supports random:
 
     crontab -l | { cat; echo "0 0 * * * sleep \$[RANDOM\%600] ; $(which peeringdb) sync > /dev/null 2>&1"; } | crontab -
+
+
+## local server
+
+This document outlines the server related commands of the `peeringdb` CLI. These commands help the user to manage a local peeringdb server snapshot for `peeringdb`.
+
+### Setup server
+
+The following command will build and setup the PeeringDB server container.
+
+```sh
+$ peeringdb server --setup
+```
+
+### Start server
+
+The following command will start the PeeringDB server container.
+
+```sh
+$ peeringdb server --start
+```
+
+### Stop server
+
+The following command will stop the PeeringDB server container.
+
+```sh
+$ peeringdb server --stop
+```
+
+## Error Handling
+
+If you try to start or stop the server without running the setup first, you will see an error stating that the `peeringdb_server` directory was not found. If this happens, run the `--setup` command and then try again.

pyproject.toml

@@ -10,7 +10,21 @@ readme = "README.md"
 repository = "https://github.com/peeringdb/peeringdb-py"
 authors = [ "PeeringDB <[email protected]>",]
 license = "Apache-2.0"
-classifiers = [ "Development Status :: 5 - Production/Stable", "Framework :: Django :: 2.2", "Framework :: Django :: 3.0", "Framework :: Django :: 3.1", "Framework :: Django :: 3.2", "Intended Audience :: Developers", "Intended Audience :: System Administrators", "Intended Audience :: Telecommunications Industry", "License :: OSI Approved :: Apache Software License", "Programming Language :: Python :: 3.6", "Programming Language :: Python :: 3.7", "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", "Topic :: Software Development :: Libraries :: Python Modules", "Topic :: Internet",]
+classifiers = [ 
+    "Development Status :: 5 - Production/Stable",  
+    "Framework :: Django :: 3.2", 
+    "Framework :: Django :: 4.2", 
+    "Intended Audience :: Developers", 
+    "Intended Audience :: System Administrators", 
+    "Intended Audience :: Telecommunications Industry", 
+    "License :: OSI Approved :: Apache Software License", 
+    "Programming Language :: Python :: 3.8", 
+    "Programming Language :: Python :: 3.9", 
+    "Programming Language :: Python :: 3.10", 
+    "Programming Language :: Python :: 3.11", 
+    "Topic :: Software Development :: Libraries :: Python Modules",
+    "Topic :: Internet"
+]
 [[tool.poetry.packages]]
 include = "peeringdb"
 from = "src"
@@ -23,18 +37,17 @@ multi_line_output = 3
 peeringdb = "peeringdb.cli:main"
 
 [tool.poetry.dependencies]
-python = "^3.7"
+python = "^3.8"
 confu = "^1"
 munge = { extras = ["tomlkit", "yaml"], version = "^1.2.0" }
 "twentyc.rpc" = "^1"
 
 [tool.poetry.dev-dependencies]
 
 # testing
-django = "~3.2"
-django_peeringdb = "^2.16"
+django = "~4.2"
+django_peeringdb = "^3.0.0"
 
-codecov = "*"
 pytest = "^6.0.1"
 pytest-cov = "*"
 tox = ">=3.24"

src/peeringdb/_fetch.py

@@ -1,7 +1,11 @@
 """
 RPC / REST client implementation module
 """
+import json
+import logging
+import os
 import re
+import time
 
 import requests
 from twentyc.rpc import RestClient
@@ -26,6 +30,17 @@ class Fetcher(RestClient):
     def __init__(self, **kwargs):
         # self.return_error = True
         self.api_key = kwargs.get("api_key", "")
+        self.cache_url = kwargs.get("cache_url")
+        self.cache_dir = os.path.expanduser(
+            kwargs.get("cache_dir", "~/.cache/peeringdb")
+        )
+        self._log = logging.getLogger(__name__)
+
+        self.cache_downloaded = None
+        self.cache_file_used = None
+
+        if not os.path.exists(self.cache_dir):
+            os.makedirs(self.cache_dir)
         super().__init__(**kwargs)
 
     def _req(self, func):
@@ -60,12 +75,65 @@ def fetch_all(self, R, depth, params={}):
         }
         return self._req(lambda: self.all(R.tag, depth=depth, **params))
 
+    def _fetch_cache(self, R):
+        """
+        Will fetch the latest version of the object from the cache
+        This is only done on empty databases (e.g., when all the data needs
+        to be fetched)
+
+        It will also use a local file cache that lasts 15 minutes.
+        """
+        ref_tag = R.tag
+        cache_url = self.cache_url
+
+        # check if we have an existing cache file
+
+        cache_file = os.path.join(self.cache_dir, f"{ref_tag}-0.json")
+        if os.path.exists(cache_file):
+            # get file modification time
+            mtime = os.path.getmtime(cache_file)
+
+            # if not older than 15 mins return from file
+            if time.time() - mtime < 15 * 60:
+                self._log.debug(f"Using cached file {cache_file}")
+                with open(cache_file) as f:
+                    # this is currently only used for testing purposes (so we can easily
+                    # check if the cache file was used)
+                    self.cache_file_used = True
+
+                    # return cache file contents
+                    return json.load(f)["data"]
+
+        # no file cache, load data from cache url
+        url = f"{cache_url}/{ref_tag}-0.json"
+        self._log.debug(f"Downloading from cache: {url}")
+        response = requests.get(url)
+
+        # this is currently only used for testing purposes (so we can easily
+        # check if the cache download was used)
+        self.cache_downloaded = True
+
+        if response.status_code == 200:
+            # write cache file
+            with open(os.path.join(self.cache_dir, f"{ref_tag}-0.json"), "w") as f:
+                f.write(response.text)
+
+            # return data
+            return response.json()["data"]
+        else:
+            raise Exception(f"Failed to download JSON file for {ref_tag}")
+
     def fetch_all_latest(self, R, depth, params={}, since=None):
         backend = peeringdb.get_backend()
 
         if since is None:
             since = backend.last_change(backend.get_concrete(R))
 
+        if not since and self.cache_url and not params and not depth:
+            # empty database, do full data update from cache server
+
+            return self._req(lambda: self._fetch_cache(R))
+
         if since:
             since = since + 1
         params = {

src/peeringdb/_update.py

@@ -171,7 +171,11 @@ def fetch_and_index(self, fetch_func):
     def update_after(self, res, pk, depth, fetch_job):
         data = yield fetch_job
         if data:
-            row = data[pk]
+            try:
+                row = data[pk]
+            except KeyError:
+                print("Data", data)
+                raise
         else:
             self._log.info("Fetched no data for %s-%s", res.tag, pk)
             data, e = self.fetcher.fetch_deleted(res, pk, 0)

src/peeringdb/cli.py

@@ -18,6 +18,7 @@
         help="Configuration management",
     ),
     "drop-tables": commands.DropTables,
+    "server": commands.Server,
 }
 
 

src/peeringdb/commands.py

@@ -1,4 +1,6 @@
 import logging
+import os
+import subprocess
 import sys
 
 import munge
@@ -257,3 +259,66 @@ def handle(config, **_):
         Client(config)
         B = peeringdb.get_backend()
         B.delete_all()
+
+
+class Server:
+    "Configure Peeringdb Server"
+
+    @staticmethod
+    def add_arguments(parser):
+        parser.add_argument(
+            "--setup",
+            action="store_true",
+            default=False,
+            help="Build and setup peeringdb server container",
+        )
+        parser.add_argument(
+            "--start",
+            action="store_true",
+            default=False,
+            help="Start peeringdb server container",
+        )
+        parser.add_argument(
+            "--stop",
+            action="store_true",
+            default=False,
+            help="Stop peeringdb server container",
+        )
+
+    @_handler
+    def handle(config, setup, start, stop, **_):
+        parent_directory = os.path.abspath(os.path.join(os.getcwd()))
+        clone_path = os.path.join(parent_directory, "peeringdb_server")
+
+        if setup:
+            # Clone the GitHub repository
+            # TODO: use latest release? (peeringdb server currently does no publish releases, just tags)
+            # TODO: use git module instead of subprocess?
+            print("Setup-----------")
+            subprocess.run(
+                [
+                    "git",
+                    "clone",
+                    "https://github.com/peeringdb/peeringdb.git",
+                    clone_path,
+                ]
+            )
+
+            # Run setup.sh inside the cloned repository
+            subprocess.run(["./Ctl/local/setup.sh"], cwd=clone_path)
+
+        if start:
+            try:
+                subprocess.run(["./Ctl/local/compose.sh", "up", "-d"], cwd=clone_path)
+            except FileNotFoundError:
+                print(
+                    f"{clone_path} directory not found, make sure that you already run 'peeringdb server --setup'"
+                )
+
+        if stop:
+            try:
+                subprocess.run(["./Ctl/local/compose.sh", "down"], cwd=clone_path)
+            except FileNotFoundError:
+                print(
+                    f"{clone_path} directory not found, make sure that you already run 'peeringdb server --setup'"
+                )

src/peeringdb/config.py

@@ -22,6 +22,8 @@ class ClientSchema(_schema.Schema):
 
     class SyncSchema(_schema.Schema):
         url = _schema.Url("url", default="https://www.peeringdb.com/api")
+        cache_url = _schema.Url("url", default="https://cache.peeringdb.com/api")
+        cache_dir = _schema.Str("cache_dir", default="~/.cache/peeringdb")
         user = _schema.Str("user", blank=True, default="")
         password = _schema.Str("password", blank=True, default="")
         strip_tz = _schema.Int("strip_tz", default=1)  # FIXME no boolean?

tests/data/cache/campus-0.json

@@ -0,0 +1,28 @@
+{
+    "data": [
+        {
+            "id": 1,
+            "org_id": 1,
+            "org_name": "org c1ab780b",
+            "status": "ok",
+            "created": "2023-04-21T13:46:31.660560Z",
+            "updated": "2023-04-21T16:02:46.336666Z",
+            "name": "campus 41c40b88",
+            "name_long": "campus 505fd8c1",
+            "notes": "Some notes",
+            "aka": "campus 34d1fb8d",
+            "website": "https://www.peeringdb.com",
+            "social_media": [
+                {
+                    "service": "website",
+                    "identifier": "https://www.peeringdb.com"
+                }
+            ],
+            "country": "US",
+            "city": "Chicago",
+            "zipcode": "12345",
+            "state": "Illinois"
+        }
+    ],
+    "meta": {}
+}