initial pass

Author: asg017

.gitignore

@@ -10,3 +10,5 @@ ggml-metal.metal
 News*.json
 sqlite-lembed.h
 dist/
+
+*.db

Makefile

@@ -85,6 +85,11 @@ $(TARGET_LOADABLE): sqlite-lembed.c sqlite-lembed.h $(BUILD_DIR) $(prefix)
 	ls $(BUILD_DIR)
 	cp $(BUILT_LOADABLE_PATH) $@
 
+$(TARGET_STATIC): sqlite-lembed.c sqlite-lembed.h $(BUILD_DIR) $(prefix)
+	cmake --build $(BUILD_DIR) -t sqlite_lembed_static $(EXTRA_CMAKE_BUILD)
+	ls $(BUILD_DIR)
+	cp $(BUILT_LOADABLE_PATH) $@
+
 
 sqlite-lembed.h: sqlite-lembed.h.tmpl VERSION
 	VERSION=$(shell cat VERSION) \

README.md

@@ -17,8 +17,8 @@ To load it into `sqlite-lembed`, register it with the `temp.lembed_models` table
 ```sql
 .load ./lembed0
 
-INSERT INTO temp.lembed_models(name, model)
-  select 'all-MiniLM-L6-v2', lembed_model_from_file('all-MiniLM-L6-v2.e4ce9877.q8_0.gguf');
+insert into temp.lembed_models(name, model)
+  values ('default', lembed_model_from_file('all-MiniLM-L6-v2.e4ce9877.q8_0.gguf'));
 
 select lembed(
   'all-MiniLM-L6-v2',
@@ -54,7 +54,7 @@ create virtual table vec_articles using vec0(
 );
 
 insert into vec_articles(rowid, headline_embeddings)
-  select rowid, lembed('all-MiniLM-L6-v2', headline)
+  select rowid, lembed( headline)
   from articles;
 
 ```
@@ -71,7 +71,7 @@ with matches as (
     rowid,
     distance
   from vec_articles
-  where headline_embeddings match lembed('all-MiniLM-L6-v2', :query)
+  where headline_embeddings match lembed(:query)
   order by distance
   limit 3
 )

bench/batch.sh

@@ -0,0 +1 @@
+#!/bin/bash

debug.sql

@@ -0,0 +1,46 @@
+.load ./dist/lembed0
+.load ../sqlite-vec/dist/vec0
+
+select lembed_version(), lembed_debug();
+
+insert into temp.lembed_models(name, model)
+  select 'default', lembed_model_from_file('');
+
+select vec_to_json(vec_slice(lembed('Shohei Ohtani''s ex-interpreter pleads guilty to charges related to gambling and theft'), 0, 8));
+select vec_to_json(vec_slice(lembed('The jury has been selected in Hunter Biden''s gun trial'), 0, 8));
+
+
+.mode box
+.header on
+
+select
+  rowid,
+  contents,
+  --length(embedding),
+  vec_to_json(vec_slice(embedding, 0, 8)),
+  vec_to_json(vec_slice(lembed(contents), 0, 8))
+
+from lembed_batch(
+  (
+    '[
+      {"contents": "Shohei Ohtani''s ex-interpreter pleads guilty to charges related to gambling and theft"}
+    ]'
+  )
+);
+
+
+select
+  rowid,
+  contents,
+  --length(embedding),
+  vec_to_json(vec_slice(embedding, 0, 8)),
+  vec_to_json(vec_slice(lembed(contents), 0, 8))
+
+from lembed_batch(
+  (
+    '[
+      {"contents": "Shohei Ohtani''s ex-interpreter pleads guilty to charges related to gambling and theft"},
+      {"contents": "The jury has been selected in Hunter Biden''s gun trial"}
+    ]'
+  )
+);