downgrate to python 3.10 and pytorch 2.0.0

iloveconference · Oct 12, 2023 · f8c1afc · f8c1afc
1 parent 8ba3f94
commit f8c1afc
Show file tree

Hide file tree

Showing 8 changed files with 519 additions and 184 deletions.
diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml
@@ -12,14 +12,14 @@ jobs:
       fail-fast: false
       matrix:
         include:
-          - { python: "3.11", os: "ubuntu-latest", session: "pre-commit" }
-          - { python: "3.11", os: "ubuntu-latest", session: "safety" }
-          - { python: "3.11", os: "ubuntu-latest", session: "mypy" }
-          - { python: "3.11", os: "ubuntu-latest", session: "tests" }
-          - { python: "3.11", os: "macos-latest", session: "tests" }
-          - { python: "3.11", os: "ubuntu-latest", session: "typeguard" }
-          - { python: "3.11", os: "ubuntu-latest", session: "xdoctest" }
-          - { python: "3.11", os: "ubuntu-latest", session: "docs-build" }
+          - { python: "3.10", os: "ubuntu-latest", session: "pre-commit" }
+          - { python: "3.10", os: "ubuntu-latest", session: "safety" }
+          - { python: "3.10", os: "ubuntu-latest", session: "mypy" }
+          - { python: "3.10", os: "ubuntu-latest", session: "tests" }
+          - { python: "3.10", os: "macos-latest", session: "tests" }
+          - { python: "3.10", os: "ubuntu-latest", session: "typeguard" }
+          - { python: "3.10", os: "ubuntu-latest", session: "xdoctest" }
+          - { python: "3.10", os: "ubuntu-latest", session: "docs-build" }
 
     env:
       NOXSESSION: ${{ matrix.session }}
@@ -112,7 +112,7 @@ jobs:
       - name: Set up Python
         uses: actions/[email protected]
         with:
-          python-version: "3.11"
+          python-version: "3.10"
 
       - name: Upgrade pip
         run: |

diff --git a/.rtx.toml b/.rtx.toml
@@ -1,4 +1,4 @@
 [tools]
-python = "3.11"
+python = "3.10"
 poetry = "1.6.1"
 pipx = "latest"
diff --git a/README.md b/README.md
@@ -50,12 +50,19 @@ Activate the poetry virtual environment: `poetry shell`
 
 Periodically add the files you are working on to git and run `nox` to run all checks and tests as you develop.
 
-If nox fails, you can run the individual checks and tests manually; e.g., `nox -s pre-commit`, `nox -s mypy-3.11`, or `nox -s tests-3.11`
+If nox fails, you can run the individual checks and tests manually; e.g., `nox -s pre-commit`, `nox -s mypy-3.10`, or `nox -s tests-3.10`
 
 Run `nox` before creating a pull request to ensure that all checks pass.
 
 ### Running notebooks
 
+After running `poetry shell`, you need to install the poetry virtual environment as a jupyter kernel.
+Let's name it "models": `python -m ipykernel install --user --name models`
+You only need to do this once.
+
+You can run notebooks either in VS Code, or in your browser.
+To run notebooks in the browser, you run
+
 `` env PYTHONPATH=`pwd` jupyter notebook ``
 
 or (if you have fish shell)

diff --git a/models/crawl_utils.py b/models/crawl_utils.py
@@ -1,13 +1,18 @@
 """Crawl utils."""
 import json
+import time
 from typing import Optional
 from typing import Tuple
 
 import requests
 
 
 def get_page(
-    url: str, headers: Optional[dict[str, str]] = None, encoding: str = "utf-8", timeout: int = 30
+    url: str,
+    delay_seconds: int = 30,
+    headers: Optional[dict[str, str]] = None,
+    encoding: str = "utf-8",
+    timeout: int = 30,
 ) -> Tuple[int, str]:
     """Get page from url."""
     if headers is None:
@@ -26,6 +31,7 @@ def get_page(
             "User-Agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/105.0.0.0 Safari/537.36",  # noqa: B950
         }
     response = requests.get(url, headers=headers, timeout=timeout)
+    time.sleep(delay_seconds)
     if encoding:
         response.encoding = encoding
     return response.status_code, response.text

diff --git a/notebooks/05_conference_crawler.ipynb b/notebooks/05_conference_crawler.ipynb
@@ -27,7 +27,6 @@
    "outputs": [],
    "source": [
     "import os\n",
-    "import time\n",
     "from urllib.parse import urljoin, urlparse\n",
     "\n",
     "from bs4 import BeautifulSoup\n",
@@ -48,7 +47,7 @@
     "host = 'https://www.churchofjesuschrist.org'\n",
     "base_dir = '../data/load/raw'\n",
     "bs_parser = 'html.parser'\n",
-    "seconds_delay = 30"
+    "delay_seconds = 30"
    ]
   },
   {
@@ -86,24 +85,22 @@
     "for year in years:\n",
     "    for month in months:\n",
     "        dir_url = f\"{host}/study/general-conference/{year}/{month}?lang=eng\"\n",
-    "        status_code, dir_html = get_page(dir_url)\n",
+    "        status_code, dir_html = get_page(dir_url, delay_seconds)\n",
     "        if status_code != 200:\n",
     "            print(f\"Status code={status_code} url={dir_url}\")\n",
     "            continue\n",
     "        talk_urls = get_talk_urls(dir_url, dir_html)\n",
     "        print(dir_url, len(talk_urls))\n",
-    "        time.sleep(seconds_delay)\n",
     "        for talk_url in talk_urls:\n",
     "            path = get_talk_path(talk_url)\n",
     "            if os.path.exists(path):\n",
     "                continue\n",
     "            print(\"    \", path)\n",
-    "            status_code, talk_html = get_page(talk_url)\n",
+    "            status_code, talk_html = get_page(talk_url, delay_seconds)\n",
     "            if status_code != 200:\n",
     "                print(f\"Status code={status_code} url={talk_url}\")\n",
     "                continue\n",
-    "            save_page(path, talk_url, talk_html)\n",
-    "            time.sleep(seconds_delay)"
+    "            save_page(path, talk_url, talk_html)"
    ]
   },
   {

diff --git a/noxfile.py b/noxfile.py
@@ -23,7 +23,7 @@
 
 
 package = "models"
-python_versions = ["3.11"]
+python_versions = ["3.10"]
 nox.needs_version = ">= 2021.6.6"
 nox.options.sessions = (
     "pre-commit",