ref(similarity): Catch all seer exceptions in backfill (#75104)

Catch all seer exceptions in retry in backfill

Author: jangjodi

src/sentry/tasks/embeddings_grouping/utils.py

@@ -352,12 +352,12 @@ def _make_seer_call(
             create_grouping_records_request,
             retries=3,
             delay=2,
-            exceptions=ServiceUnavailable,
+            exceptions=Exception,
         )
-    except ServiceUnavailable:
+    except Exception as e:
         logger.exception(
-            "tasks.backfill_seer_grouping_records.seer_service_unavailable",
-            extra={"project_id": project_id},
+            "tasks.backfill_seer_grouping_records.seer_exception_after_retries",
+            extra={"project_id": project_id, "error": e},
         )
         raise
 

tests/sentry/tasks/test_backfill_seer_grouping_records.py

@@ -5,7 +5,7 @@
 from random import choice
 from string import ascii_uppercase
 from typing import Any
-from unittest.mock import call, patch
+from unittest.mock import ANY, call, patch
 
 import pytest
 from django.test import override_settings
@@ -241,15 +241,15 @@ def test_lookup_group_data_stacktrace_bulk_exceptions(
             mock_get_multi.side_effect = exception
             with pytest.raises(Exception):
                 lookup_group_data_stacktrace_bulk(self.project, rows)
-                mock_logger.exception.assert_called_with(
-                    "tasks.backfill_seer_grouping_records.bulk_event_lookup_exception",
-                    extra={
-                        "organization_id": self.project.organization.id,
-                        "project_id": self.project.id,
-                        "group_data": json.dumps(rows),
-                        "error": exception.message,
-                    },
-                )
+            mock_logger.exception.assert_called_with(
+                "tasks.backfill_seer_grouping_records.bulk_event_lookup_exception",
+                extra={
+                    "organization_id": self.project.organization.id,
+                    "project_id": self.project.id,
+                    "node_keys": ANY,
+                    "error": exception.message,
+                },
+            )
 
     def test_lookup_group_data_stacktrace_bulk_not_stacktrace_grouping(self):
         """
@@ -446,7 +446,7 @@ def test_get_data_from_snuba(self):
             assert group_id in group_ids_results
 
     @patch("sentry.tasks.embeddings_grouping.utils.logger")
-    @patch("sentry.utils.snuba.bulk_snuba_queries")
+    @patch("sentry.tasks.embeddings_grouping.utils.bulk_snuba_queries")
     def test_get_data_from_snuba_exception(self, mock_bulk_snuba_queries, mock_logger):
         mock_bulk_snuba_queries.side_effect = RateLimitExceeded
 
@@ -455,14 +455,14 @@ def test_get_data_from_snuba_exception(self, mock_bulk_snuba_queries, mock_logge
         }
         with pytest.raises(Exception):
             get_data_from_snuba(self.project, group_ids_last_seen)
-            mock_logger.exception.assert_called_with(
-                "tasks.backfill_seer_grouping_records.snuba_query_exception",
-                extra={
-                    "organization_id": self.project.organization.id,
-                    "project_id": self.project.id,
-                    "error": "Snuba Rate Limit Exceeded",
-                },
-            )
+        mock_logger.exception.assert_called_with(
+            "tasks.backfill_seer_grouping_records.snuba_query_exception",
+            extra={
+                "organization_id": self.project.organization.id,
+                "project_id": self.project.id,
+                "error": "Snuba Rate Limit Exceeded",
+            },
+        )
 
     @with_feature("projects:similarity-embeddings-backfill")
     @patch("sentry.tasks.embeddings_grouping.utils.post_bulk_grouping_records")
@@ -1541,6 +1541,29 @@ def test_backfill_seer_grouping_records_no_enable_ingestion(
 
         assert self.project.get_option("sentry:similarity_backfill_completed") is None
 
+    @with_feature("projects:similarity-embeddings-backfill")
+    @patch("time.sleep", return_value=None)
+    @patch("sentry.tasks.embeddings_grouping.utils.logger")
+    @patch("sentry.tasks.embeddings_grouping.utils.post_bulk_grouping_records")
+    def test_backfill_seer_grouping_records_seer_exception(
+        self, mock_post_bulk_grouping_records, mock_logger, mock_sleep
+    ):
+        """
+        Test log after seer exception and retries.
+        """
+        exception = ServiceUnavailable(message="Service Unavailable")
+        mock_post_bulk_grouping_records.side_effect = exception
+        with pytest.raises(Exception), TaskRunner():
+            backfill_seer_grouping_records_for_project(self.project.id, None)
+
+        mock_logger.exception.assert_called_with(
+            "tasks.backfill_seer_grouping_records.seer_exception_after_retries",
+            extra={
+                "project_id": self.project.id,
+                "error": exception,
+            },
+        )
+
     @with_feature("projects:similarity-embeddings-backfill")
     @patch("sentry.tasks.embeddings_grouping.backfill_seer_grouping_records_for_project.logger")
     def test_backfill_seer_grouping_records_skip_project_already_processed(self, mock_logger):