Merge branch 'master' into build-docker-image

Author: tclem

.travis.yml

@@ -28,14 +28,16 @@ before_install:
 - cabal --version
 
 install:
-- cabal new-update hackage.haskell.org,HEAD
+- cabal new-update hackage.haskell.org
 - cabal new-configure --enable-tests --write-ghc-environment-files=always
 - cabal new-build --only-dependencies -j
 
 script:
-- cabal new-build -j semantic-core semantic:exe:semantic
-- cabal new-test semantic:test semantic-core:spec
-- cabal new-run semantic:parse-examples
+- cabal new-build -j
+- cabal new-run semantic:test
+- cabal new-run semantic-core:spec
+# parse-examples is disabled because it slaughters our CI
+# - cabal new-run semantic:parse-examples
 
 # Any branch linked with a pull request will be built, as well as the non-PR
 # branches listed below:

cabal.project

@@ -2,6 +2,9 @@ packages: . semantic-core
 
 jobs: $ncpus
 
+package semantic
+  ghc-options: +RTS -A128m -n2m -RTS
+
 source-repository-package
   type: git
   location: https://github.com/joshvera/proto3-suite.git

docs/development.md

@@ -1,18 +1,22 @@
 # Development Guide
 
-`semantic` is built using the [`stack`](https://github.com/commercialhaskell/stack) build system and the [ghc](https://www.haskell.org/ghc/) compiler. It’s packaged using [Cabal](https://www.haskell.org/cabal/), and uses dependencies from [Hackage](http://hackage.haskell.org/).
+`semantic` is compiled with the [ghc](https://www.haskell.org/ghc/) compiler and built/packaged with [`cabal`](https://cabal.readthedocs.io/en/latest/). It uses Cabal's [Nix-style local builds](https://www.haskell.org/cabal/users-guide/nix-local-build-overview.html) and uses dependencies from [Hackage](http://hackage.haskell.org/). We recommend using [`ghcup`](https://www.haskell.org/ghcup/) to sandbox your installation of GHC.
 
 | Tool | Explanation |
 | :-------------: |-------------|
 | [`ghc`](https://www.haskell.org/ghc/) | The Glasgow Haskell Compiler is the open source compiler and interactive environment for the Haskell programming language. |
 | [`cabal`](https://www.haskell.org/cabal/) | Cabal is a system for building and packaging Haskell libraries and programs. It is similar to having `make` files instead of having to type several complicated calls to `ghc` to compile and link a project. |
-| [`stack`](https://docs.haskellstack.org/en/stable/README/) | Stack is used to develop Haskell projects. It is a layer on top of `cabal` and `ghc` that aims to make it much easier to build cross-platform Haskell projects. It lets us use different versions of `ghc` and has snapshots of packages that are known to work together. It defers to `cabal` for much of its work. Historically, we used `cabal` directly, but ran into a variety of tooling issues that [brought us back to stack](https://github.com/github/semantic/pull/1335). Officially, `stack` uses [stackage](https://www.stackage.org/) as its package archive, and `cabal-install` uses hackage. |
 | [`hackage`](https://hackage.haskell.org/) | Hackage is the most widely used package archive of open source libraries and programs. `cabal-install` is used to download and install packages. |
+| [`ghcup`](https://www.haskell.org/ghcup/) | `ghcup` takes care of managing different versions of GHC on your system. We aggressively track new versions of GHC; sandboxing makes upgrading to new versions safe and easy. |
 | [`ghci`](https://downloads.haskell.org/~ghc/5.04/docs/html/users_guide/ghci.html) | `ghci` is GHC's interactive environment. This is where Haskell expressions can be interactively evaluated and programs can be interpreted. |
 
+### Nix-style local builds
+
+`semantic` is a complicated app with a very large dependency tree. Because managing large dependency trees in a system-wide `ghc` installation is difficult, especially when developing on multiple Haskell projects, `cabal` enables "local" builds: each dependency is linked in per-project, not globally. In practice, this means that you should prefix your commands with the `new-` prefix: `cabal new-build` builds the project, `new-clean` purges its build artifacts, etc. (With versions of the `cabal` command line tool newer than 2.6, local builds become the default, with the `v1-` prefix required to yield old behavior.)
+
 ### Running a REPL
 
-Running `stack ghci semantic` will boot a GHCi with the right environment for Semantic set up.
+Running `cabal new-repl semantic:lib` will boot a GHCi with the right environment for Semantic set up.
 
 See the [💡ProTips](💡ProTip!.md#ghci) for more info.
 

script/clone-example-repos

@@ -13,7 +13,10 @@
 set -e
 cd $(dirname "$0")/..
 
-dir="vendor/haskell-tree-sitter/languages"
+mkdir -p test/examplerepos || true
+git clone --single-branch --recurse-submodules https://github.com/tree-sitter/haskell-tree-sitter.git tmp/haskell-tree-sitter || true
+
+dir="tmp/haskell-tree-sitter/languages"
 
 # clone_repo LOCAL_PATH URL SHA
 function clone_repo {

semantic.cabal

@@ -49,7 +49,7 @@ common dependencies
                      , bytestring ^>= 0.10.8.2
                      , containers ^>= 0.6.0.1
                      , directory ^>= 1.3.3.0
-                     , fastsum
+                     , fastsum ^>= 0.1.1.0
                      , filepath ^>= 1.4.2.1
                      , free ^>= 5.1
                      , fused-effects ^>= 0.4.0.0
@@ -59,11 +59,11 @@ common dependencies
                      , machines ^>= 0.6.4
                      , mtl ^>= 2.2.2
                      , network ^>= 2.8.0.0
-                     , process
+                     , process ^>= 1.6.3.0
                      , recursion-schemes ^>= 5.1
                      , scientific ^>= 0.3.6.2
-                     , safe-exceptions
-                     , semilattices
+                     , safe-exceptions ^>= 0.1.7.0
+                     , semilattices ^>= 0.0.0.3
                      , text ^>= 1.2.3.1
                      , these >= 0.7 && <1
                      , unix ^>= 2.7.2.2
@@ -309,11 +309,11 @@ library
                      , stm-chans ^>= 3.0.0.4
                      , template-haskell ^>= 2.14
                      , time ^>= 1.8.0.2
-                     , unliftio-core
+                     , unliftio-core ^>= 0.1.2.0
                      , unordered-containers ^>= 0.2.9.0
                      , vector ^>= 0.12.0.2
                      , tree-sitter-go ^>= 0.1.0.0
-                     , tree-sitter-haskell
+                     , tree-sitter-haskell ^>= 0.1.0.0
                      , tree-sitter-json ^>= 0.1.0.0
                      , tree-sitter-php ^>= 0.1.0.0
                      , tree-sitter-python ^>= 0.1.0.1

src/Semantic/Git.hs

@@ -9,12 +9,19 @@ module Semantic.Git
   , ObjectType(..)
   , ObjectMode(..)
   , OID(..)
+
+  -- Testing Purposes
+  , parseEntries
+  , parseEntry
   ) where
 
 import Control.Monad.IO.Class
-import Data.Text as Text
-import Shelly hiding (FilePath)
-import System.IO (hSetBinaryMode)
+import Data.Attoparsec.Text   (Parser)
+import Data.Attoparsec.Text   as AP
+import Data.Char
+import Data.Text              as Text
+import Shelly                 hiding (FilePath)
+import System.IO              (hSetBinaryMode)
 
 -- | git clone --bare
 clone :: Text -> FilePath -> IO ()
@@ -24,22 +31,38 @@ clone url path = sh $ do
 -- | git cat-file -p
 catFile :: FilePath -> OID -> IO Text
 catFile gitDir (OID oid) = sh $ do
-  run "git" [pack ("--git-dir=" <> gitDir), "cat-file", "-p", oid]
+  run "git" ["-C", pack gitDir, "cat-file", "-p", oid]
 
 -- | git ls-tree -rz
 lsTree :: FilePath -> OID -> IO [TreeEntry]
-lsTree gitDir (OID sha) = sh $ do
-  out <- run "git" [pack ("--git-dir=" <> gitDir), "ls-tree", "-rz", sha]
-  pure $ mkEntry <$> splitOn "\NUL" out
-  where
-    mkEntry row | [mode, ty, rest] <- splitOn " " row
-                , [oid, path] <- splitOn "\t" rest
-                = TreeEntry (objectMode mode) (objectType ty) (OID oid) (unpack path)
-                | otherwise = nullTreeEntry
+lsTree gitDir (OID sha) = sh $ parseEntries <$> run "git" ["-C", pack gitDir, "ls-tree", "-rz", sha]
 
 sh :: MonadIO m => Sh a -> m a
 sh = shelly . silently . onCommandHandles (initOutputHandles (`hSetBinaryMode` True))
 
+-- | Parses an list of entries separated by \NUL, and on failure return []
+parseEntries :: Text -> [TreeEntry]
+parseEntries = either (const []) id . AP.parseOnly everything
+  where
+    everything = AP.sepBy entryParser "\NUL" <* "\NUL\n" <* AP.endOfInput
+
+-- | Parse the entire input with entryParser, and on failure return a default
+-- For testing purposes only
+parseEntry :: Text -> Either String TreeEntry
+parseEntry = AP.parseOnly (entryParser <* AP.endOfInput)
+
+-- | Parses a TreeEntry
+entryParser :: Parser TreeEntry
+entryParser = TreeEntry
+  <$> modeParser <* AP.char ' '
+  <*> typeParser <* AP.char ' '
+  <*> oidParser <* AP.char '\t'
+  <*> (unpack <$> AP.takeWhile (/= '\NUL'))
+    where
+      typeParser = AP.choice [BlobObject <$ "blob", TreeObject <$ "tree"]
+      modeParser = AP.choice [NormalMode <$ "100644", ExecutableMode <$ "100755", SymlinkMode <$ "120000", TreeMode <$ "040000"]
+      oidParser = OID <$> AP.takeWhile isHexDigit
+
 newtype OID = OID Text
   deriving (Eq, Show, Ord)
 
@@ -51,24 +74,12 @@ data ObjectMode
   | OtherMode
   deriving (Eq, Show)
 
-objectMode :: Text -> ObjectMode
-objectMode "100644" = NormalMode
-objectMode "100755" = ExecutableMode
-objectMode "120000" = SymlinkMode
-objectMode "040000" = TreeMode
-objectMode _        = OtherMode
-
 data ObjectType
   = BlobObject
   | TreeObject
   | OtherObjectType
   deriving (Eq, Show)
 
-objectType :: Text -> ObjectType
-objectType "blob" = BlobObject
-objectType "tree" = TreeObject
-objectType _      = OtherObjectType
-
 data TreeEntry
   = TreeEntry
   { treeEntryMode :: ObjectMode
@@ -77,5 +88,3 @@ data TreeEntry
   , treeEntryPath :: FilePath
   } deriving (Eq, Show)
 
-nullTreeEntry :: TreeEntry
-nullTreeEntry = TreeEntry OtherMode OtherObjectType (OID mempty) mempty

test/Analysis/Go/Spec.hs

@@ -1,3 +1,4 @@
+{-# OPTIONS_GHC -O0 #-}
 module Analysis.Go.Spec (spec) where
 
 import Data.Abstract.Evaluatable (EvalError(..))

test/Analysis/PHP/Spec.hs

@@ -1,3 +1,4 @@
+{-# OPTIONS_GHC -O0 #-}
 module Analysis.PHP.Spec (spec) where
 
 import           Control.Abstract

test/Analysis/Ruby/Spec.hs

@@ -1,3 +1,4 @@
+{-# OPTIONS_GHC -O0 #-}
 {-# LANGUAGE TupleSections #-}
 module Analysis.Ruby.Spec (spec) where
 

test/Analysis/TypeScript/Spec.hs

@@ -1,3 +1,5 @@
+{-# OPTIONS_GHC -O0 #-}
+
 module Analysis.TypeScript.Spec (spec) where
 
 import           Data.Syntax.Statement (StatementBlock(..))