Merge pull request #293 from galgeek/prom-ii

update prometheus metrics, approved separately by @avdempsey

brozzler/metrics.py

@@ -17,13 +17,18 @@
 from prometheus_client import Counter, Gauge, Histogram, start_http_server
 
 # fmt: off
-brozzler_pages_crawled = Counter("brozzler_pages_crawled", "number of pages visited by brozzler")
+brozzler_in_progress_pages = Gauge("brozzler_in_progress_pages", "number of pages currently processing with brozzler")
 brozzler_page_processing_duration_seconds = Histogram("brozzler_page_processing_duration_seconds", "time spent processing a page in brozzler")
+brozzler_in_progress_headers = Gauge("brozzler_in_progress_headers", "number of headers currently processing with brozzler")
+brozzler_header_processing_duration_seconds = Histogram("brozzler_header_processing_duration_seconds", "time spent processing one page's headers in brozzler")
+brozzler_in_progress_browses = Gauge("brozzler_in_progress_browses", "number of pages currently browsing with brozzler")
+brozzler_browsing_duration_seconds = Histogram("brozzler_browsing_duration_seconds", "time spent browsing a page in brozzler")
+brozzler_in_progress_ytdlps = Gauge("brozzler_in_progress_ytdlps", "number of ytdlp sessions currently in progress with brozzler")
+brozzler_ytdlp_duration_seconds = Histogram("brozzler_ytdlp_duration_seconds", "time spent running ytdlp for a page in brozzler")
+brozzler_pages_crawled = Counter("brozzler_pages_crawled", "number of pages visited by brozzler")
 brozzler_outlinks_found = Counter("brozzler_outlinks_found", "number of outlinks found by brozzler")
 brozzler_last_page_crawled_time = Gauge("brozzler_last_page_crawled_time", "time of last page visit, in seconds since UNIX epoch")
-brozzler_in_progress_pages = Gauge("brozzler_in_progress_pages", "number of pages currently processing with brozzler")
 brozzler_ydl_urls_checked = Counter("brozzler_ydl_urls_checked", "count of urls checked by brozzler yt-dlp")
-brozzler_ydl_extract_attempts = Counter("brozzler_ydl_extract_attempts", "count of extracts attempted by brozzler yt-dlp", labelnames=["youtube_host"])
 brozzler_ydl_extract_successes = Counter("brozzler_ydl_extract_successes", "count of extracts completed by brozzler yt-dlp", labelnames=["youtube_host"])
 brozzler_ydl_download_successes = Counter("brozzler_ydl_download_successes", "count of downloads completed by brozzler yt-dlp", labelnames=["youtube_host"])
 # fmt: on

brozzler/worker.py

@@ -251,6 +251,8 @@ def thumb_jpeg(self, full_jpeg):
         img.save(out, "jpeg", quality=95)
         return out.getbuffer()
 
+    @metrics.brozzler_page_processing_duration_seconds.time()
+    @metrics.brozzler_in_progress_pages.track_inprogress()
     def brozzle_page(
         self,
         browser,
@@ -315,6 +317,8 @@ def brozzle_page(
                         )
         return outlinks
 
+    @metrics.brozzler_header_processing_duration_seconds.time()
+    @metrics.brozzler_in_progress_headers.track_inprogress()
     def _get_page_headers(self, page):
         # bypassing warcprox, requests' stream=True defers downloading the body of the response
         # see https://docs.python-requests.org/en/latest/user/advanced/#body-content-workflow
@@ -334,8 +338,8 @@ def _needs_browsing(self, page_headers):
             return False
         return True
 
-    @metrics.brozzler_page_processing_duration_seconds.time()
-    @metrics.brozzler_in_progress_pages.track_inprogress()
+    @metrics.brozzler_browsing_duration_seconds.time()
+    @metrics.brozzler_in_progress_browses.track_inprogress()
     def _browse_page(self, browser, site, page, on_screenshot=None, on_request=None):
         def update_page_metrics(page, outlinks):
             """Update page-level Prometheus metrics."""

brozzler/ydl.py

@@ -334,7 +334,6 @@ def _try_youtube_dl(worker, ydl, site, page):
             # ydl.extract_info(str(urlcanon.whatwg(ydl.url)), download=should_download_vid)
             # if ydl.is_youtube_host and ie_result:
             #     download_url = ie_result.get("url")
-            metrics.brozzler_ydl_extract_attempts.labels(ydl.is_youtube_host).inc(1)
             with brozzler.thread_accept_exceptions():
                 # we do whatwg canonicalization here to avoid "<urlopen error
                 # no host given>" resulting in ProxyError
@@ -406,6 +405,8 @@ def _try_youtube_dl(worker, ydl, site, page):
     return ie_result
 
 
+@metrics.brozzler_ytdlp_duration_seconds.time()
+@metrics.brozzler_in_progress_ytdlps.track_inprogress()
 def do_youtube_dl(worker, site, page):
     """
     Runs yt-dlp configured for `worker` and `site` to download videos from