Fix datasource add and delete issues (#147)

Author: vegito22

llmstack/apps/tasks.py

@@ -30,9 +30,10 @@ def delete_data_entry_task(
         datasource_entry_items = datasource_entry_handler.delete_entry(
             entry_data.config,
         )
-        logger.debug(
-            f"Deleted {len(datasource_entry_items)} items from weaviate for data_source_entry: {str(entry_data.uuid)}",
-        )
+        if datasource_entry_items:
+            logger.debug(
+                f"Deleted {len(datasource_entry_items)} items from weaviate for data_source_entry: {str(entry_data.uuid)}",
+            )
         entry_data.delete()
     except weaviate.exceptions.UnexpectedStatusCodeException:
         logger.exception("Error deleting data source entry from weaviate")

llmstack/datasources/handlers/website/url.py

@@ -7,7 +7,6 @@
 from llmstack.common.blocks.data.store.vectorstore import Document
 from llmstack.common.utils.splitter import SpacyTextSplitter
 from llmstack.common.utils.text_extract import ExtraParams, extract_text_from_url
-from llmstack.common.utils.utils import extract_urls_from_sitemap
 from llmstack.datasources.handlers.datasource_processor import (
     WEAVIATE_SCHEMA,
     DataSourceEntryItem,
@@ -116,25 +115,13 @@ def get_url_data(
 
     def validate_and_process(self, data: dict) -> List[DataSourceEntryItem]:
         entry = URLSchema(**data)
-        sitemap_urls = []
         # Split urls by newline and then by comma
         urls = entry.urls.split("\n")
         urls = [url.strip().rstrip() for url_list in [url.split(",") for url in urls] for url in url_list]
         # Filter out empty urls
         urls = list(set(list(filter(lambda url: url != "", urls))))
-        sitemap_xmls = list(
-            filter(lambda url: url.endswith(".xml"), urls),
-        )
         # Filter out sitemap.xml
         urls = list(filter(lambda url: not url.endswith(".xml"), urls))
-        # If sitemap.xml is present, scrape the site to extract urls
-        try:
-            for sitemap_xml in sitemap_xmls:
-                sitmap_xml_urls = extract_urls_from_sitemap(sitemap_xml)
-                for sitmap_xml_url in sitmap_xml_urls:
-                    sitemap_urls.append(sitmap_xml_url)
-        except BaseException:
-            logger.exception("Error in extracting urls from sitemap")
 
         return list(
             map(
@@ -145,7 +132,7 @@ def validate_and_process(self, data: dict) -> List[DataSourceEntryItem]:
                         "connection_id": entry.connection_id,
                     },
                 ),
-                urls + sitemap_urls,
+                urls,
             ),
         )
 

llmstack/jobs/models.py

@@ -758,6 +758,15 @@ def save(self, *args, **kwargs):
             on_failure = kwargs.pop("on_failure", None)
             job_meta = kwargs.pop("job_meta", None)
 
+        update_fields = kwargs.get("update_fields", None)
+        if update_fields:
+            kwargs["update_fields"] = set(
+                update_fields,
+            ).union({"updated_at"})
+
+        super(AdhocJob, self).save(*args, **kwargs)
+
+        if schedule_job:
             job = self.rqueue.enqueue(
                 func,
                 args=func_args,
@@ -770,13 +779,6 @@ def save(self, *args, **kwargs):
             )
             self.job_id = job.id
 
-        update_fields = kwargs.get("update_fields", None)
-        if update_fields:
-            kwargs["update_fields"] = set(
-                update_fields,
-            ).union({"updated_at"})
-        super(AdhocJob, self).save(*args, **kwargs)
-
     class Meta:
         verbose_name = "Adhoc Job"
         verbose_name_plural = "Adhoc Jobs"