Added

Author: djthorpe

cmd/indexer/main.go

@@ -0,0 +1,112 @@
+package main
+
+import (
+	"context"
+	"flag"
+	"fmt"
+	"os"
+	"os/signal"
+	"strings"
+	"sync"
+	"syscall"
+	"time"
+	"unicode"
+
+	"github.com/mutablelogic/go-sqlite/pkg/indexer"
+)
+
+var (
+	flagInclude = flag.String("include", "", "Paths, names and extensions to include")
+	flagExclude = flag.String("exclude", "", "Paths, names and extensions to exclude")
+)
+
+func main() {
+	flag.Parse()
+	if flag.NArg() != 1 {
+		fmt.Fprintln(os.Stderr, "missing path argument")
+		os.Exit(-1)
+	}
+	path := flag.Arg(0)
+	if stat, err := os.Stat(path); err != nil {
+		fmt.Fprintln(os.Stderr, err)
+		os.Exit(-1)
+	} else if !stat.IsDir() {
+		fmt.Fprintln(os.Stderr, "Not a directory")
+		os.Exit(-1)
+	}
+	indexer, err := indexer.NewIndexer("indexer", path)
+	if err != nil {
+		fmt.Fprintln(os.Stderr, err)
+		os.Exit(-1)
+	}
+
+	// Inclusions
+	for _, include := range strings.FieldsFunc(*flagInclude, sep) {
+		if err := indexer.Include(include); err != nil {
+			fmt.Fprintln(os.Stderr, err)
+			os.Exit(-1)
+		}
+	}
+
+	// Exclusions
+	for _, exclude := range strings.FieldsFunc(*flagExclude, sep) {
+		if err := indexer.Exclude(exclude); err != nil {
+			fmt.Fprintln(os.Stderr, err)
+			os.Exit(-1)
+		}
+	}
+
+	var wg sync.WaitGroup
+
+	ctx := HandleSignal()
+	errs := make(chan error)
+	wg.Add(1)
+	go func() {
+		defer wg.Done()
+		for {
+			select {
+			case err := <-errs:
+				fmt.Fprintln(os.Stderr, err)
+			case <-ctx.Done():
+				return
+			}
+		}
+	}()
+
+	wg.Add(1)
+	go func() {
+		defer wg.Done()
+		if err := indexer.Run(ctx, errs); err != nil {
+			errs <- err
+		}
+	}()
+
+	wg.Add(1)
+	go func() {
+		defer wg.Done()
+		<-time.After(time.Second)
+		if err := indexer.Walk(ctx); err != nil {
+			errs <- err
+		}
+	}()
+
+	// Wait for all goroutines to finish
+	wg.Wait()
+	os.Exit(0)
+}
+
+func HandleSignal() context.Context {
+	// Handle signals - call cancel when interrupt received
+	ctx, cancel := context.WithCancel(context.Background())
+	ch := make(chan os.Signal, 1)
+	signal.Notify(ch, os.Interrupt, syscall.SIGTERM)
+	go func() {
+		<-ch
+		cancel()
+	}()
+	return ctx
+}
+
+func sep(r rune) bool {
+	return r == ',' || unicode.IsSpace(r)
+}

pkg/indexer/indexer.go

@@ -74,7 +74,7 @@ func (i *Indexer) Run(ctx context.Context, errs chan<- error) error {
 
 	in := make(chan notify.EventInfo, defaultCapacity)
 	if err := notify.Watch(filepath.Join(i.path, "..."), in, notify.Create, notify.Remove, notify.Write, notify.Rename); err != nil {
-		fmt.Println("err", err)
+		senderr(errs, err)
 		return err
 	}
 
@@ -156,13 +156,34 @@ func (i *Indexer) Walk(ctx context.Context) error {
 
 // event is used to process an event from the notify
 func (i *Indexer) event(ctx context.Context, evt notify.EventInfo) error {
-	fmt.Println("event", evt)
+	relpath, err := filepath.Rel(i.path, evt.Path())
+	if err != nil {
+		return err
+	}
+	switch evt.Event() {
+	case notify.Create, notify.Write:
+		info, err := os.Stat(evt.Path())
+		if err == nil && info.Mode().IsRegular() && i.ShouldVisit(relpath, info) {
+			fmt.Println("INDEX ", relpath)
+		}
+	case notify.Remove, notify.Rename:
+		info, err := os.Stat(evt.Path())
+		if err == nil && info.Mode().IsRegular() && i.ShouldVisit(relpath, info) {
+			fmt.Println("INDEX ", relpath)
+		} else {
+			// Always attempt removal from index
+			fmt.Println("REMOVE ", relpath)
+		}
+	}
+	// Return success
 	return nil
 }
 
 // visit is used to index a file from the indexer
 func (i *Indexer) visit(ctx context.Context, abspath, relpath string, info fs.FileInfo) error {
-	fmt.Println("visited", abspath, relpath, info.IsDir())
+	if info.Mode().IsRegular() {
+		fmt.Println("INDEX ", relpath)
+	}
 	return nil
 }
 
@@ -178,73 +199,3 @@ func senderr(ch chan<- error, err error) {
 		}
 	}
 }
-
-/*
-// Return an indexer event or nil if no event should be sent
-func (this *Indexer) process(e EventType, path string, info fs.FileInfo, out chan<- IndexerEvent, block bool) error {
-	// Normalize the path
-	relpath, err := filepath.Rel(this.path, path)
-	if err != nil {
-		return err
-	} else {
-		relpath = pathSeparator + relpath
-	}
-
-	// Deal with exclusions
-	if e&EVENT_TYPE_ADDED > 0 {
-		// Check for path exclusions
-		for exclusion := range this.expath {
-			if strings.HasPrefix(relpath, exclusion) {
-				return nil
-			}
-		}
-		// Check for extension exclusions
-		if info != nil && info.Mode().IsRegular() {
-			ext := strings.ToUpper(filepath.Ext(info.Name()))
-			if _, exists := this.exext[ext]; exists {
-				return nil
-			}
-		}
-	}
-
-	// Send event
-	if block {
-		out <- NewEvent(e, this.name, relpath, info)
-	} else {
-		select {
-		case out <- NewEvent(e, this.name, relpath, info):
-			// No-op
-		default:
-			return ErrChannelBlocked.With(this.name)
-		}
-	}
-
-	// Return success
-	return nil
-}
-
-// Translate notify types to internal types
-func toEventType(e notify.Event, info fs.FileInfo) EventType {
-	switch e {
-	case notify.Create:
-		if info != nil {
-			return EVENT_TYPE_ADDED
-		}
-	case notify.Remove:
-		return EVENT_TYPE_REMOVED
-	case notify.Rename:
-		if info != nil {
-			return EVENT_TYPE_ADDED | EVENT_TYPE_RENAMED
-		} else {
-			return EVENT_TYPE_REMOVED | EVENT_TYPE_RENAMED
-		}
-	case notify.Write:
-		if info != nil {
-			return EVENT_TYPE_ADDED | EVENT_TYPE_CHANGED
-		}
-	}
-
-	// Ignore unhandled cases
-	return EVENT_TYPE_NONE
-}
-*/

pkg/indexer/indexer_test.go

@@ -10,7 +10,7 @@ import (
 )
 
 const (
-	TEST_PATH_1 = "../../.."
+	TEST_PATH_1 = "../../../.."
 )
 
 func Test_Indexer_000(t *testing.T) {
@@ -48,7 +48,12 @@ func Test_Indexer_001(t *testing.T) {
 	}()
 
 	// Queue up an indexing operation
-	indexer.Exclude("/waveshare")
+	indexer.Exclude("/Library")
+	indexer.Exclude("/go")
+	indexer.Exclude("/Work")
+	indexer.Exclude("/bin")
+	indexer.Exclude("/Projects")
+	indexer.Exclude("/Documents")
 	if err := indexer.Walk(ctx); err != nil {
 		t.Error(err)
 	}

pkg/indexer/queue.go

@@ -0,0 +1,98 @@
+package indexer
+
+import (
+	"io/fs"
+	"path/filepath"
+	"sync"
+	// Package imports
+	// Import namepaces
+	//. "github.com/djthorpe/go-errors"
+)
+
+///////////////////////////////////////////////////////////////////////////////
+// TYPES
+
+type Queue struct {
+	sync.RWMutex
+	q []QueueEvent
+	k map[string]*QueueEvent
+}
+
+type QueueEvent struct {
+	name string
+	path string
+	info fs.FileInfo
+}
+
+///////////////////////////////////////////////////////////////////////////////
+// GLOBALS
+
+const (
+	defaultQueueCapacity = 1024
+)
+
+///////////////////////////////////////////////////////////////////////////////
+// LIFECYCLE
+
+// Create a new queue which acts as a buffer between the file indexing
+func NewQueue() *Queue {
+	return NewQueueWithCapacity(defaultQueueCapacity)
+}
+
+// Create a new queue which acts as a buffer between the file indexing
+// and the rendering which can be slower than the file indexing
+func NewQueueWithCapacity(cap int) *Queue {
+	q := new(Queue)
+
+	// Create capacity
+	if cap == 0 {
+		q.q = make([]QueueEvent, 0, defaultQueueCapacity)
+		q.k = make(map[string]*QueueEvent, defaultQueueCapacity)
+	} else {
+		q.q = make([]QueueEvent, 0, cap)
+		q.k = make(map[string]*QueueEvent, cap)
+	}
+
+	// Return success
+	return q
+}
+
+///////////////////////////////////////////////////////////////////////////////
+// PUBLIC METHODS
+
+// Add an item to the queue. If the item is already in the queue,
+// then it is bumped to the end of the queue
+func (q *Queue) Add(name, path string) error {
+	/*	if elem := q.Get(name, path); elem != nil {
+			// Remove the element from the existing queue
+			q.pop(elem)
+		}
+		// Add the element to the queue
+		q.push(elem)*/
+	return nil
+}
+
+// Remove an item to the queue. If the item is already in the queue,
+// it is removed
+func (q *Queue) Remove(name, path string) error {
+	/*	if elem := q.Get(name, path); elem != nil {
+			// Remove the element from the existing queue
+			q.pop(elem)
+		} else {
+			return ErrNotFound.With(name, path)
+		}*/
+	// Return success
+	return nil
+}
+
+// Return a queue event from the queue, or nil
+func (q *Queue) Get(name, path string) *QueueEvent {
+	q.RWMutex.RLock()
+	defer q.RWMutex.RUnlock()
+	key := filepath.Join(name, path)
+	if elem, exists := q.k[key]; exists {
+		return elem
+	} else {
+		return nil
+	}
+}