Regression fix: Update failed crawl database object after deleting files (#2993)

Fixes #2991 

After deleting files (e.g. WACZs uploaded while a crawl was paused) for
canceled or otherwise failed crawls, ensure we also update the crawl
database object.

This fixes a regression introduced by crawl pausing, which resulted in
org storage numbers being incorrect when later deleting the canceled
crawl as a consequence of the crawl files not having been deleted from
the database at the same time as they were deleted from storage.

It also renames the basecrawls `delete_crawl_files` method to
`delete_failed_crawl_files` to make purpose clearer, as it is only used
by the operator and should only be used for failed crawls (when deleting
successful crawls, there are other workflow- and org-related updates
that are handled by other codepaths).

Author: tw4l

backend/btrixcloud/basecrawls.py

@@ -426,11 +426,21 @@ async def _delete_crawl_files(
 
         return size
 
-    async def delete_crawl_files(self, crawl_id: str, oid: UUID):
-        """Delete crawl files"""
+    async def delete_failed_crawl_files(self, crawl_id: str, oid: UUID):
+        """Delete crawl files for failed crawl"""
         crawl = await self.get_base_crawl(crawl_id)
         org = await self.orgs.get_org_by_id(oid)
-        return await self._delete_crawl_files(crawl, org)
+        await self._delete_crawl_files(crawl, org)
+        await self.crawls.find_one_and_update(
+            {"_id": crawl_id, "oid": oid},
+            {
+                "$set": {
+                    "files": [],
+                    "fileCount": 0,
+                    "fileSize": 0,
+                }
+            },
+        )
 
     async def delete_all_crawl_qa_files(self, crawl_id: str, org: Organization):
         """Delete files for all qa runs in a crawl"""

backend/btrixcloud/operator/crawls.py

@@ -1707,7 +1707,7 @@ async def do_crawl_finished_tasks(
                 )
 
         if state in FAILED_STATES:
-            await self.crawl_ops.delete_crawl_files(crawl.id, crawl.oid)
+            await self.crawl_ops.delete_failed_crawl_files(crawl.id, crawl.oid)
             await self.page_ops.delete_crawl_pages(crawl.id, crawl.oid)
 
         await self.event_webhook_ops.create_crawl_finished_notification(