Ajouter les règlements de Sainte-Agathe-des-Monts (#30)

* fix: supprimer des scripts inutilises

* fix: reparer des erreurs dans les reglements de sainte-agathe

* fix: ne pas reecrire un csv existant

* chore: retrain

* feat: telecharger sainte-agathe aussi

* fix: lint

.github/workflows/analyse.yml

@@ -52,12 +52,15 @@ jobs:
                 fi
             done
         done
+        alexi -v download -u https://vsadm.ca/citoyens/reglementation/reglementation-durbanisme/index.html -o download/vsadm --all-pdf-links
     - name: Extract
       run: |
         alexi -v extract -m download/index.json download/*.pdf
+        alexi -v extract -m download/vsadm/index.json -o export/vsadm download/vsadm/*.pdf
     - name: Index
       run: |
         alexi -v index export
+        alexi -v index export/vsadm
     - name: Setup Pages
       uses: actions/configure-pages@v5
     - name: Upload artifact

TODO.md

@@ -3,7 +3,10 @@ DATA
 
 - Correct titles in zonage glossary
 - Correct extraction (see below, use RNN) of titles numbers etc
-- Annotate multiple TOCs in Sainte-Agathe urbanisme
+- Annotate multiple TOCs in Sainte-Agathe urbanisme DONE
+- Add Sainte-Agathe to download DONE
+- Add Sainte-Agathe to export (under /vsadm) DONE
+- Do the same thing for Saint-Sauveur
 
 DERP LERNING
 ------------

alexi/annotate.py

@@ -140,6 +140,10 @@ def main(args: argparse.Namespace) -> None:
         with open(args.csv, "rt", encoding="utf-8-sig") as infh:
             iob = list(csv.DictReader(infh))
     else:
+        args.csv = args.out.with_suffix(".csv")
+        if args.csv.exists():
+            LOGGER.error("Fichier déjà existant: %s", args.csv)
+            return
         if args.segment_model is not None:
             crf = Segmenteur(args.segment_model)
             crf_n = crf
@@ -155,7 +159,7 @@ def main(args: argparse.Namespace) -> None:
         else:
             segs = crf(feats)
         iob = list(spread_i(crf_s(segs)))
-        with open(args.out.with_suffix(".csv"), "wt") as outfh:
+        with open(args.csv, "wt") as outfh:
             write_csv(iob, outfh)
     annotate_pdf(args.doc, pages, iob, args.out.with_suffix(".pdf"))
 

alexi/download.py

@@ -39,6 +39,12 @@ def add_arguments(parser: argparse.ArgumentParser) -> argparse.ArgumentParser:
         action="append",
         default=[],
     )
+    parser.add_argument(
+        "--all-pdf-links",
+        action="store_true",
+        help="Télécharger les liens vers des PDF dans le document "
+        "sans égard à sa structure",
+    )
     parser.add_argument(
         "section",
         help="Expression régulière pour sélectionner la section des documents",
@@ -71,19 +77,25 @@ def main(args: argparse.Namespace) -> None:
     paths = []
     with open(args.outdir / Path(u.path).name) as infh:
         soup = BeautifulSoup(infh, "lxml")
-        for h2 in soup.find_all("h2", string=re.compile(args.section, re.I)):
-            ul = h2.find_next("ul")
-            for li in ul.find_all("li"):
-                path = li.a["href"]
-                excluded = False
-                for r in excludes:
-                    if r.search(path):
-                        excluded = True
-                        break
-                if not excluded:
+        if args.all_pdf_links:
+            for a in soup.find_all("a"):
+                path = a["href"]
+                if path.lower().endswith(".pdf"):
                     paths.append(path)
+        else:
+            for h2 in soup.find_all("h2", string=re.compile(args.section, re.I)):
+                ul = h2.find_next("ul")
+                for li in ul.find_all("li"):
+                    paths.append(li.a["href"])
     urls = {}
     for p in paths:
+        excluded = False
+        for r in excludes:
+            if r.search(p):
+                excluded = True
+                break
+        if excluded:
+            continue
         up = urllib.parse.urlparse(p)
         if up.netloc:
             url = p

scripts/comparables.sh

@@ -1,7 +1,8 @@
-wget --no-check-certificate --timestamping --recursive --level=1 https://www.val-morin.ca/reglementation_formulaires/reglementation_urbanisme.php
-wget --no-check-certificate --timestamping --recursive --level=1 https://www.valdavid.com/services-aux-citoyens/urbanisme-et-permis/reglements-durbanisme/
-wget --no-check-certificate --timestamping --recursive --level=1 https://www.piedmont.ca/fr/administration-municipale/reglements-municipaux/
-wget --no-check-certificate --timestamping --recursive --level=1 https://www.vss.ca/services-aux-citoyens/services/reglementation-durbanisme
-wget --no-check-certificate --timestamping --recursive --level=1 https://www.sadl.qc.ca/reglements-urbanisme/
-wget --no-check-certificate --timestamping --recursive --level=1 https://www.morinheights.com/Reglements
-wget --no-check-certificate --timestamping --recursive --level=1 https://lacmasson.com/services-aux-citoyens/plan-durbanisme
+wget --accept pdf --no-check-certificate --timestamping --recursive --level=1 https://vsadm.ca/citoyens/reglementation/reglementation-durbanisme/
+wget --accept pdf --no-check-certificate --timestamping --recursive --level=1 https://www.val-morin.ca/reglementation_formulaires/reglementation_urbanisme.php
+wget --accept pdf --no-check-certificate --timestamping --recursive --level=1 https://www.valdavid.com/services-aux-citoyens/urbanisme-et-permis/reglements-durbanisme/
+wget --accept pdf --no-check-certificate --timestamping --recursive --level=1 https://www.piedmont.ca/fr/administration-municipale/reglements-municipaux/
+wget --accept pdf --no-check-certificate --timestamping --recursive --level=1 https://www.vss.ca/services-aux-citoyens/services/reglementation-durbanisme
+wget --accept pdf --no-check-certificate --timestamping --recursive --level=1 https://www.sadl.qc.ca/reglements-urbanisme/
+wget --accept pdf --no-check-certificate --timestamping --recursive --level=1 https://www.morinheights.com/Reglements
+wget --accept pdf --no-check-certificate --timestamping --recursive --level=1 https://lacmasson.com/services-aux-citoyens/plan-durbanisme