Fix datasource/app deletion workflow error (#10)

* Change order of deletions to ensure we don't hit foreign key constraints error on app or datasource deletion

* Use queryset delete to delete run_graph_entries

Author: vegito22

apps/apis.py

@@ -301,18 +301,18 @@ def delete(self, request, uid):
 
         # Cleanup app run_graph
         run_graph_entries = app.run_graph.all()
-        endpoint_entries = filter(lambda x: x != None, set(list(map(lambda x: x.entry_endpoint, run_graph_entries)) +
-                                                           list(map(lambda x: x.exit_endpoint, run_graph_entries))))
+        endpoint_entries = list(filter(lambda x: x != None, set(list(map(lambda x: x.entry_endpoint, run_graph_entries)) +
+                                                           list(map(lambda x: x.exit_endpoint, run_graph_entries)))))
 
+        # Cleanup rungraph 
+        # Delete all the run_graph entries
+        run_graph_entries.delete()
+            
         # Delete all the endpoint entries
         for entry in endpoint_entries:
             EndpointViewSet.delete(self, request, id=str(
                 entry.parent_uuid), force_delete_app=True)
 
-        # Delete all the run_graph entries
-        for entry in run_graph_entries:
-            entry.delete()
-
         app.delete()
 
         return DRFResponse(status=200)

apps/tasks.py

@@ -1,6 +1,5 @@
 import logging
 import uuid
-from typing import Any
 from typing import List
 
 from datasources.handlers.datasource_type_interface import DataSourceEntryItem
@@ -9,6 +8,7 @@
 from datasources.models import DataSourceEntry
 from datasources.models import DataSourceEntryStatus
 from datasources.types import DataSourceTypeFactory
+import weaviate
 
 logger = logging.getLogger(__name__)
 
@@ -65,11 +65,23 @@ def delete_data_entry_task(datasource: DataSource, entry_data: DataSourceEntry):
         datasource.type,
     )
     datasource_entry_handler = datasource_entry_handler_cls(datasource)
-    datasource_entry_items = datasource_entry_handler.delete_entry(
-        entry_data.config,
-    )
-
-    entry_data.delete()
+    try:
+        datasource_entry_items = datasource_entry_handler.delete_entry(
+            entry_data.config,
+        )
+        entry_data.delete()
+    except weaviate.exceptions.UnexpectedStatusCodeException:
+        logger.exception("Error deleting data source entry from weaviate")
+        entry_data.delete()
+    except Exception as e:
+        logger.exception(
+            f'Error deleting data_source_entry: %s' %
+            str(entry_data.name),
+        )
+        entry_data.status = DataSourceEntryStatus.FAILED
+        entry_data.config = {'errors': {'message':"Error in deleting data source entry"}}
+        entry_data.save()
+    
     datasource.save()
     return datasource_entry_items
 

datasources/apis.py

@@ -3,8 +3,6 @@
 from urllib.parse import urlparse
 
 from django.shortcuts import get_object_or_404
-from django.utils.decorators import method_decorator
-from django.views.decorators.cache import cache_page
 from rest_framework import viewsets
 from rest_framework.response import Response as DRFResponse
 
@@ -127,9 +125,16 @@ def delete(self, request, uid):
             DataSource, uuid=uuid.UUID(uid), owner=request.user,
         )
 
-        # Delete the data source and cleanup all data source entries associated with it
+        # Delete all datasource entries associated with the datasource
+        datasource_entries = DataSourceEntry.objects.filter(
+            datasource=datasource,
+        )
+        for entry in datasource_entries:
+            DataSourceEntryViewSet().delete(request=request, uid=str(entry.uuid))
+        
+        # Delete the data from data store
         delete_data_source_task(datasource)
-
+        
         datasource.delete()
         return DRFResponse(status=204)