Refactor in-memory storage to external storage through SQLite (#42)

Decouple data load from app startup in preparation for remote sourcing

Author: arvyy

README.adoc

@@ -132,6 +132,10 @@ The following is exhaustive list of valid properties
 | Which port to use. Relevant only if enable-web is `#t`
 | 8080
 
+| db-path
+| Path to DB file used by SQLite
+| "db"
+
 | spec-index
 | Index of definitions to load (see Identifiers definitions section)
 | "./types/index.scm"

build.sbt

@@ -12,9 +12,7 @@ libraryDependencies += ("org.apache.solr" % "solr-core" % solrVersion)
   .exclude("org.apache.logging.log4j", "log4j-slf4j-impl")
   .exclude("org.apache.logging.log4j", "log4j-core")
 
-libraryDependencies += "com.fasterxml.jackson.module" %% "jackson-module-scala" % "2.14.1"
-
-val logbackVersion = "1.2.6"
+val logbackVersion = "1.4.7"
 libraryDependencies += "ch.qos.logback" % "logback-core" % logbackVersion
 libraryDependencies += "ch.qos.logback" % "logback-classic" % logbackVersion
 
@@ -26,8 +24,12 @@ libraryDependencies ++= Seq(
 )
 
 libraryDependencies += "io.circe" %% "circe-generic" % "0.14.3"
-
+libraryDependencies += "com.fasterxml.jackson.module" %% "jackson-module-scala" % "2.14.1"
+libraryDependencies += "org.xerial" % "sqlite-jdbc" % "3.41.2.1"
+libraryDependencies += "com.zaxxer" % "HikariCP" % "5.0.1"
 libraryDependencies += "org.scalatest" %% "scalatest" % "3.2.15" % "test"
+libraryDependencies += "org.tpolecat" %% "doobie-core" % "1.0.0-RC1"
+libraryDependencies += "org.flywaydb" % "flyway-core" % "9.17.0"
 
 scalacOptions ++= Seq("-Xmax-inlines", "1000")
 

src/main/resources/db/migration/V1__initial_setup.sql

@@ -0,0 +1,41 @@
+create table filterset(
+    id integer primary key,
+    code text,
+    name text
+);
+
+create table filterset_library(
+    id integer primary key,
+    filterset_id integer,
+    name text,
+
+    foreign key(filterset_id) references filterset(id)
+);
+
+create table index_entry(
+    id integer primary key,
+    group_id integer,
+    lib text,
+    name text,
+    signature text,
+    description text,
+
+    foreign key(group_id) references index_entry(id)
+);
+
+create table index_entry_tag(
+    id integer primary key,
+    index_entry_id integer,
+    name text,
+
+    foreign key(index_entry_id) references index_entry(id)
+);
+
+create table index_entry_subsignature(
+    id integer primary key,
+    index_entry_id integer,
+    name text,
+    signature text,
+
+    foreign key(index_entry_id) references index_entry(id)
+);

src/main/scala/scmindex/Config.scala

@@ -1,12 +1,14 @@
 package scmindex
 
+import scmindex.core.*
 import java.nio.charset.StandardCharsets
 import java.nio.file.{Files, Path}
 import cats.effect.IO
 import org.slf4j.LoggerFactory
 
 case class Config(
   port: Int,
+  dbPath: String,
   specIndex: String,
   solrEmbed: Boolean,
   solrHome: String,
@@ -43,25 +45,31 @@ object Config {
     for {
       map <- Sexpr.alistToMap(sexpr)
       port <- getInt(map, "port", 8080)
+      dbPath <- getString(map, "db-path", "db")
       specIndex <- getString(map, "spec-index", "types/index.scm")
       solrEmbed <- getBool(map, "solr-embed", true)
       solrHome <- getString(map, "solr-home", "./solrhome")
       solrUrl <- getString(map, "solr-url", "http://localhost:8983/solr")
       solrCore <- getString(map, "solr-core", "scmindex")
-      filtersetIndex <- getString(map, "filterset-idnex", "filters/index.scm")
-    } yield Config(port, specIndex, solrEmbed, solrHome, solrUrl, solrCore, filtersetIndex)
+      filtersetIndex <- getString(map, "filterset-index", "filters/index.scm")
+    } yield Config(port, dbPath, specIndex, solrEmbed, solrHome, solrUrl, solrCore, filtersetIndex)
   }
 
-  given SignatureLoader[Config] with {
+  given Importer[Config] with {
     extension (config: Config) {
       def loadIndex() = config.loadLibrary(config.specIndex)
-
       def loadLibrary(file: String) = IO {
         val path = Path.of(file)
         val content = Files.readString(path, StandardCharsets.UTF_8)
         log.info(s"Loading library ${file}")
         SexprParser.read(content)
       }
+      def loadFiltersetIndex() = config.loadLibrary(config.filtersetIndex)
+      def loadFilterset(file: String) = IO {
+        val path = Path.of(file)
+        val content = Files.readString(path, StandardCharsets.UTF_8)
+        SexprParser.read(content)
+      }
     }
   }
 

src/main/scala/scmindex/Indexer.scala

@@ -4,29 +4,38 @@ import java.nio.charset.StandardCharsets
 import java.nio.file.{Files, Path}
 import cats.data.EitherT
 import cats.effect.{ExitCode, IO, IOApp}
-import scmindex.SolrIndexer.given
+import scmindex.persistance.*
+import scmindex.persistance.SolrIndexer.given
+import scmindex.core.*
+import scmindex.web.WebController
+import scmindex.Config.given
 
 object Main extends IOApp {
 
   def run(args: List[String]): IO[ExitCode] = {
+
     val configFile = args match {
       case Nil => "./config/configuration.scm"
       case x :: _ => x
     }
 
-    {for {
+    val modelEitherT = for {
       configSexpr <- EitherT(SexprParser.readFromFile(configFile))
       cfg <- EitherT.fromEither(Config.readFromSexpr(configSexpr))
-      entriesList <- EitherT(SCMIndexEntry.loadSignatures(cfg))
-      entries = entriesList.toVector
-      filtersetsSexpr <- EitherT(SexprParser.readFromFile(cfg.filtersetIndex))
-      filtersetsReader = (str: String) => SexprParser.readFromFile(str)
-      filtersets <- EitherT(Filterset.loadFiltersets(filtersetsSexpr, filtersetsReader))
-      solrIndexer = if (cfg.solrEmbed) SolrIndexer.createEmbeddedSolrIndexer(cfg.solrHome, cfg.solrCore) else SolrIndexer.createRemoteSolrIndexer(cfg.solrUrl)
-      _ <- EitherT(solrIndexer.index(entries).map {_ => Right(())})
-      model = Model(solrIndexer, cfg, filtersets, entries)
-    } yield model}.value.flatMap {
-      case Right(model) => WebController.runServer(model)
+      solrIndexer =
+        if (cfg.solrEmbed) 
+          SolrIndexer.createEmbeddedSolrIndexer(cfg.solrHome, cfg.solrCore)
+        else 
+          SolrIndexer.createRemoteSolrIndexer(cfg.solrUrl)
+      storage <- EitherT.liftF(SqliteStorage.create(cfg.dbPath))
+      _ <- EitherT.liftF(storage.init())
+      service = SCMIndexService(solrIndexer, storage)
+      _ <- EitherT(SCMIndexService.runImport(service, cfg))
+      exitCode <- EitherT.liftF(WebController.runServer(service, cfg.port))
+    } yield exitCode
+
+    modelEitherT.value.flatMap {
+      case Right(exitCode) => IO(exitCode)
       case Left(err) => IO(err.printStackTrace()).as(ExitCode(1))
     }
   }

src/main/scala/scmindex/Main.scala

@@ -1,4 +1,4 @@
-package scmindex
+package scmindex.core
 
 import cats.data.EitherT
 import cats.effect.IO
@@ -8,24 +8,26 @@ case class Filterset(code: String, name: String, libs: List[String])
 
 object Filterset {
 
-  def loadFiltersets(index: Sexpr, reader: (String) => IO[Either[Exception, Sexpr]]): IO[Either[Exception, List[Filterset]]] = {
-    Sexpr.sexprToProperList(index).map { lst =>
-      lst
-        .map({parseFilterset(_, reader)})
-        .sequence
-        .map { f =>
-          f.partitionMap(identity) match {
-            case (Nil, rights) => Right(rights)
-            case (first :: _, _) => Left(Exception("Failed to parse filtersets", first))
+  def loadFiltersets[T: Importer](importer: T): IO[Either[Exception, List[Filterset]]] = {
+    val eitherT = for {
+      index <- EitherT(importer.loadFiltersetIndex())
+      indexAsList <- EitherT.fromEither(Sexpr.sexprToProperList(index))
+      filtersets <- EitherT({
+        indexAsList
+          .map({parseFilterset(_, importer.loadFilterset)})
+          .sequence
+          .map { f =>
+            f.partitionMap(identity) match {
+              case (Nil, rights) => Right(rights)
+              case (first :: _, _) => Left(Exception("Failed to parse filtersets", first))
+            }
           }
-        }
-    } match {
-      case Right(v) => v
-      case Left(err) => IO(Left(err))
-    }
+      })
+    } yield filtersets
+    eitherT.value
   }
 
-  private def parseFilterset(sexpr: Sexpr, reader: (String) => IO[Either[Exception, Sexpr]]): IO[Either[Exception, Filterset]] = {
+  def parseFilterset(sexpr: Sexpr, reader: (String) => IO[Either[Exception, Sexpr]]): IO[Either[Exception, Filterset]] = {
     def getStringField(map: Map[String, Sexpr], field: String): Either[Exception, String] = {
       map.get(field) match {
         case Some(SexprString(value)) => Right(value)
@@ -44,7 +46,7 @@ object Filterset {
     res.value
   }
 
-  private def parseFiltersetLibs(sexpr: Sexpr): Either[Exception, List[String]] = {
+  def parseFiltersetLibs(sexpr: Sexpr): Either[Exception, List[String]] = {
     Sexpr.sexprToProperList(sexpr)
       .flatMap { terms =>
         val entries = terms.map {

src/main/scala/scmindex/Model.scala

@@ -1,12 +1,14 @@
-package scmindex
+package scmindex.core
 
 import cats.effect.IO
 
-trait SignatureLoader[A] {
+trait Importer[A] {
 
   extension (a :A) {
     def loadIndex(): IO[Either[Exception, Sexpr]]
     def loadLibrary(file: String): IO[Either[Exception, Sexpr]]
+    def loadFiltersetIndex(): IO[Either[Exception, Sexpr]]
+    def loadFilterset(src: String): IO[Either[Exception, Sexpr]]
   }
 
 }

src/main/scala/scmindex/Filterset.scala renamed to src/main/scala/scmindex/core/Filterset.scala

@@ -0,0 +1,15 @@
+package scmindex.core
+
+import cats.effect.IO
+
+case class FacetValue(value: String, count: Int)
+case class IndexerResponse[R](total: Int, libs: List[FacetValue], params: List[FacetValue], returns: List[FacetValue], tags: List[FacetValue], items: List[R])
+
+trait Indexer[A, ID] {
+  extension (a: A) {
+    def index(entries: List[(ID, SCMIndexEntry)]): IO[Unit]
+    def query(query: String, lib: List[String], param: List[String], returns: List[String], tags: List[String], pageSize: Int, offset: Int): IO[IndexerResponse[ID]]
+    def get(lib: String, name: String): IO[Option[ID]]
+    def listFacetOptions(lib: List[String], facetName: String): IO[List[String]]
+  }
+}