Updates

Author: djthorpe

etc/server.yaml

@@ -3,7 +3,7 @@ plugins:
   - build/httpserver.plugin
   - build/log.plugin
   - build/sqlite3.plugin
-  - build/static.plugin
+#  - build/static.plugin
 
 # HTTP Server parameters
 httpserver:
@@ -18,11 +18,11 @@ handlers:
     # Requests are logged
     middleware:
       - log
-  static:
-    prefix: /api/static
+#  static:
+#    prefix: /api/static
 
-static:
-  "/": npm/sqlite3/dist
+#static:
+#  "/": npm/sqlite3/dist
 
 sqlite3:
   # Databases to load and/or create. Only the 'main' database is required.

pkg/sqlite3/cache.go

@@ -3,22 +3,21 @@ package sqlite3
 import (
 	"sync"
 
-	// Namespace Imports
-	. "github.com/djthorpe/go-errors"
+	// Packages
 	sqlite3 "github.com/djthorpe/go-sqlite/sys/sqlite3"
 	multierror "github.com/hashicorp/go-multierror"
+
+	// Namespace Imports
+	. "github.com/djthorpe/go-errors"
 )
 
 ////////////////////////////////////////////////////////////////////////////////
 // TYPES
 
-// PoolCache caches prepared statements and profiling information for
-// statements so it's possible to see slow queries, etc.
-type PoolCache struct{}
-
 type ConnCache struct {
 	sync.Mutex
 	sync.Map
+	cap int // Capacity of the cache, defaults to 100 prepared statements
 }
 
 ////////////////////////////////////////////////////////////////////////////////
@@ -31,13 +30,17 @@ const (
 ////////////////////////////////////////////////////////////////////////////////
 // PUBLIC METHODS
 
+func (cache *ConnCache) SetCap(cap int) {
+	cache.cap = intMax(0, cap)
+}
+
 // Return a prepared statement from the cache, or prepare a new statement
 // and put it in the cache before returning
 func (cache *ConnCache) Prepare(conn *sqlite3.ConnEx, q string) (*Results, error) {
 	if conn == nil {
 		return nil, ErrInternalAppError
 	}
-	st, _ := cache.Map.Load(q)
+	st := cache.load(q)
 	if st == nil {
 		// Prepare a statement and store in cache
 		var err error
@@ -48,11 +51,8 @@ func (cache *ConnCache) Prepare(conn *sqlite3.ConnEx, q string) (*Results, error
 		} else {
 			cache.Map.Store(q, st)
 		}
-	} else {
-		// Increment counter by one
-		st.(*sqlite3.StatementEx).Inc(1)
 	}
-	return NewResults(st.(*sqlite3.StatementEx)), nil
+	return NewResults(st), nil
 }
 
 // Close all conn cache prepared statements
@@ -68,3 +68,22 @@ func (cache *ConnCache) Close() error {
 	// Return any errors
 	return result
 }
+
+////////////////////////////////////////////////////////////////////////////////
+// PRIVATE METHODS
+
+func (cache *ConnCache) load(key string) *sqlite3.StatementEx {
+	// If cache is switched off, then return nil
+	if cache.cap == 0 {
+		return nil
+	}
+	// Load statement from cache and return
+	st, _ := cache.Map.Load(key)
+	if st, ok := st.(*sqlite3.StatementEx); !ok {
+		return nil
+	} else {
+		// Increment counter by one
+		st.Inc(1)
+		return st
+	}
+}

pkg/sqlite3/cache_test.go

@@ -2,6 +2,8 @@ package sqlite3_test
 
 import (
 	"context"
+	"errors"
+	"io"
 	"math/rand"
 	"sync"
 	"testing"
@@ -16,26 +18,41 @@ import (
 )
 
 func Test_Cache_001(t *testing.T) {
-	// Create a connection
-	conn, err := OpenPath(":memory:", sqlite3.DefaultFlags)
+	// Create a connection and enable the connection cache (which caches prepared statements)
+	conn, err := OpenPath(":memory:", sqlite3.DefaultFlags|sqlite3.SQLITE_OPEN_CONNCACHE)
 	if err != nil {
 		t.Fatal(err)
 	}
 	defer conn.Close()
 
 	// Perform caching in a transaction
 	conn.Do(context.Background(), 0, func(txn SQTransaction) error {
-		// Read values from the cache
+		// SELECT n between 0-9 over 100 executions in parallel should
+		// return the same result, with a perfect cache hit rate of 9 in 10?
 		var wg sync.WaitGroup
-		for i := 0; i < 99; i++ {
+		for i := 0; i < 20; i++ {
 			wg.Add(1)
 			go func() {
 				defer wg.Done()
-				n := rand.Uint32() % 5
-				if r, err := txn.Query(Q("SELECT ", n)); err != nil {
-					t.Error(err)
-				} else {
-					t.Log(r)
+				n := rand.Uint32() % 10
+				r, err := txn.Query(Q("SELECT ", n))
+				if err != nil {
+					t.Error("Query Error: ", err)
+					return
+				}
+				for {
+					row, err := r.Next()
+					if errors.Is(err, io.EOF) {
+						break
+					} else if err != nil {
+						t.Error("Next Error: ", err)
+					} else if len(row) != 1 {
+						t.Error("Unexpected row length: ", row)
+					} else if int64(n) != row[0] {
+						t.Error("Unexpected row value: ", row, " expected [", n, "]")
+					} else {
+						t.Log(n, "=>", row[0])
+					}
 				}
 			}()
 		}

pkg/sqlite3/conn.go

@@ -39,7 +39,7 @@ type TxnFunc func(SQTransaction) error
 // LIFECYCLE
 
 func OpenPath(path string, flags sqlite3.OpenFlags) (*Conn, error) {
-	poolconn := new(Conn)
+	conn := new(Conn)
 
 	// If no create flag then check to make sure database exists
 	if path != defaultMemory && flags&sqlite3.SQLITE_OPEN_CREATE == 0 {
@@ -51,22 +51,29 @@ func OpenPath(path string, flags sqlite3.OpenFlags) (*Conn, error) {
 	}
 
 	// Open database with flags
-	if conn, err := sqlite3.OpenPathEx(path, flags, ""); err != nil {
+	if c, err := sqlite3.OpenPathEx(path, flags, ""); err != nil {
 		return nil, err
 	} else {
-		poolconn.ConnEx = conn
+		conn.ConnEx = c
+	}
+
+	// Set cache to default size
+	if flags&sqlite3.SQLITE_OPEN_CONNCACHE != 0 {
+		conn.SetCap(defaultCapacity)
+	} else {
+		conn.SetCap(0)
 	}
 
 	// Finalizer to panic when connection not closed before garbage collection
 	_, file, line, _ := runtime.Caller(1)
-	runtime.SetFinalizer(poolconn, func(conn *Conn) {
+	runtime.SetFinalizer(conn, func(conn *Conn) {
 		if conn.c != nil {
 			panic(fmt.Sprintf("%s:%d: missing associated call to Close()", file, line))
 		}
 	})
 
 	// Return success
-	return poolconn, nil
+	return conn, nil
 }
 
 func (conn *Conn) Close() error {
@@ -139,13 +146,21 @@ func (conn *Conn) Do(ctx context.Context, flag SQTxnFlag, fn func(SQTransaction)
 
 	// Commit or rollback transaction
 	if result == nil {
-		result = multierror.Append(result, conn.ConnEx.Commit())
+		if err := conn.ConnEx.Commit(); err != nil {
+			result = multierror.Append(result, err)
+		}
 	} else {
-		result = multierror.Append(result, conn.ConnEx.Rollback())
+		if err := conn.ConnEx.Rollback(); err != nil {
+			result = multierror.Append(result, err)
+		}
 	}
 
-	// Return foreign key constraints
-	result = multierror.Append(result, conn.SetForeignKeyConstraints(fk))
+	// Return foreign key constraints to previous value
+	if flag&SQLITE_TXN_NO_FOREIGNKEY_CONSTRAINTS != 0 {
+		if err := conn.SetForeignKeyConstraints(fk); err != nil {
+			result = multierror.Append(result, err)
+		}
+	}
 
 	// Return any errors
 	return result

pkg/sqlite3/pool.go

@@ -57,7 +57,7 @@ var (
 		Trace:   false,
 		Create:  true,
 		Schemas: map[string]string{defaultSchema: defaultMemory},
-		Flags:   sqlite3.SQLITE_OPEN_CREATE | sqlite3.SQLITE_OPEN_READWRITE | sqlite3.SQLITE_OPEN_SHAREDCACHE,
+		Flags:   sqlite3.SQLITE_OPEN_CREATE | sqlite3.SQLITE_OPEN_SHAREDCACHE | sqlite3.SQLITE_OPEN_CONNCACHE,
 	}
 )
 

pkg/sqlite3/results.go

@@ -4,6 +4,7 @@ import (
 	"errors"
 	"fmt"
 	"io"
+	"reflect"
 
 	// Packages
 	sqlite3 "github.com/djthorpe/go-sqlite/sys/sqlite3"
@@ -56,8 +57,42 @@ func (r *Results) NextQuery(v ...interface{}) error {
 	} else {
 		r.results = results
 		r.n++
-		// TODO: Set LastInsertId, Changes, Columns, etc.
-		fmt.Println(r.results)
+		// TODO: Set Columns, etc.
 		return nil
 	}
 }
+
+// Return a row from the results, or return io.EOF if all results have been consumed
+func (r *Results) Next(t ...reflect.Type) ([]interface{}, error) {
+	if r.results == nil {
+		return nil, io.EOF
+	} else {
+		return r.results.Next(t...)
+	}
+}
+
+func (r *Results) ExpandedSQL() string {
+	if r.results == nil {
+		return ""
+	} else {
+		return r.results.ExpandedSQL()
+	}
+}
+
+// Return LastInsertId by last execute or -1 if no valid results
+func (r *Results) LastInsertId() int64 {
+	if r.results == nil {
+		return -1
+	} else {
+		return r.results.LastInsertId()
+	}
+}
+
+// Return RowsAffected by last execute or -1 if no valid results
+func (r *Results) RowsAffected() int {
+	if r.results == nil {
+		return -1
+	} else {
+		return r.results.RowsAffected()
+	}
+}

pkg/sqlite3/util.go

@@ -44,3 +44,11 @@ func stringToBool(v string) bool {
 		return false
 	}
 }
+
+// intMax returns the maximum of two ints
+func intMax(a, b int) int {
+	if a > b {
+		return a
+	}
+	return b
+}