Merge branch 'main' into frontend-browsertrix-rename

webrecorder · Mar 11, 2024 · 2c8e80a · 2c8e80a
2 parents 6dc982e + 548261e
commit 2c8e80a
Show file tree

Hide file tree

Showing 15 changed files with 8,386 additions and 116 deletions.
diff --git a/ansible/Pipfile.lock b/ansible/Pipfile.lock
diff --git a/backend/btrixcloud/crawls.py b/backend/btrixcloud/crawls.py
@@ -441,6 +441,13 @@ async def inc_crawl_exec_time(self, crawl_id, exec_time, last_updated_time):
             },
         )
 
+    async def get_crawl_exec_last_update_time(self, crawl_id):
+        """get crawl last updated time"""
+        res = await self.crawls.find_one(
+            {"_id": crawl_id, "type": "crawl"}, projection=["_lut"]
+        )
+        return res and res.get("_lut")
+
     async def get_crawl_state(self, crawl_id):
         """return current crawl state of a crawl"""
         res = await self.crawls.find_one(

diff --git a/backend/btrixcloud/models.py b/backend/btrixcloud/models.py
@@ -522,6 +522,15 @@ class CrawlFileOut(BaseModel):
     expireAt: Optional[str]
 
 
+# ============================================================================
+class ReviewStatus(str, Enum):
+    """QA review statuses"""
+
+    GOOD = "good"
+    ACCEPTABLE = "acceptable"
+    FAILURE = "failure"
+
+
 # ============================================================================
 class BaseCrawl(BaseMongoModel):
     """Base Crawl object (representing crawls, uploads and manual sessions)"""
@@ -554,6 +563,8 @@ class BaseCrawl(BaseMongoModel):
     fileSize: int = 0
     fileCount: int = 0
 
+    reviewStatus: Optional[ReviewStatus] = None
+
 
 # ============================================================================
 class CollIdName(BaseModel):
@@ -617,6 +628,8 @@ class CrawlOut(BaseMongoModel):
     crawlerChannel: str = "default"
     image: Optional[str]
 
+    reviewStatus: Optional[ReviewStatus] = None
+
 
 # ============================================================================
 class CrawlOutWithResources(CrawlOut):
@@ -634,6 +647,7 @@ class UpdateCrawl(BaseModel):
     description: Optional[str]
     tags: Optional[List[str]]
     collectionIds: Optional[List[UUID]]
+    reviewStatus: Optional[ReviewStatus]
 
 
 # ============================================================================

diff --git a/backend/btrixcloud/operator/crawls.py b/backend/btrixcloud/operator/crawls.py
@@ -232,7 +232,9 @@ async def sync_crawls(self, data: MCSyncData):
 
         else:
             status.scale = crawl.scale
-            status.lastUpdatedTime = to_k8s_date(dt_now())
+            now = dt_now()
+            await self.crawl_ops.inc_crawl_exec_time(crawl_id, 0, now)
+            status.lastUpdatedTime = to_k8s_date(now)
 
         children = self._load_redis(params, status, data.children)
 
@@ -828,12 +830,15 @@ async def increment_pod_exec_time(
         """inc exec time tracking"""
         now = dt_now()
 
-        if not status.lastUpdatedTime:
+        update_start_time = await self.crawl_ops.get_crawl_exec_last_update_time(
+            crawl_id
+        )
+
+        if not update_start_time:
+            await self.crawl_ops.inc_crawl_exec_time(crawl_id, 0, now)
             status.lastUpdatedTime = to_k8s_date(now)
             return
 
-        update_start_time = from_k8s_date(status.lastUpdatedTime)
-
         reason = None
         update_duration = (now - update_start_time).total_seconds()
 
@@ -907,16 +912,6 @@ async def increment_pod_exec_time(
                 max_duration = max(duration, max_duration)
 
         if exec_time:
-            if not await self.crawl_ops.inc_crawl_exec_time(
-                crawl_id, exec_time, status.lastUpdatedTime
-            ):
-                # if lastUpdatedTime is same as previous, something is wrong, don't update!
-                print(
-                    "Already updated for lastUpdatedTime, skipping execTime update!",
-                    flush=True,
-                )
-                return
-
             await self.org_ops.inc_org_time_stats(oid, exec_time, True)
             status.crawlExecTime += exec_time
             status.elapsedCrawlTime += max_duration
@@ -926,6 +921,7 @@ async def increment_pod_exec_time(
             flush=True,
         )
 
+        await self.crawl_ops.inc_crawl_exec_time(crawl_id, exec_time, now)
         status.lastUpdatedTime = to_k8s_date(now)
 
     def should_mark_waiting(self, state, started):