Merge master into support-cabal-doctest

# Conflicts:
#	builder/comp-builder.nix
#	lib/check.nix
#	lib/default.nix
#	test/default.nix

Author: hamishmack

.buildkite/pipeline.yml

@@ -1,6 +1,6 @@
 steps:
-  - label: 'Run tests'
-    command: "./test/tests.sh"
+  - label: 'Run tests with ghc884'
+    command: "./test/tests.sh ghc884"
     agents:
       system: x86_64-linux
 
@@ -14,9 +14,9 @@ steps:
     agents:
       system: x86_64-linux
 
-  - label: 'Check closure size'
+  - label: 'Check closure size with ghc884'
     command:
-      - nix-build build.nix -A maintainer-scripts.check-closure-size -o check-closure-size.sh
+      - nix-build build.nix -A maintainer-scripts.check-closure-size --argstr compiler-nix-name ghc884 -o check-closure-size.sh
       - echo "+++ Closure size (MB)"
       - ./check-closure-size.sh
     agents:
@@ -28,3 +28,13 @@ steps:
       - ./update-docs.sh
     agents:
       system: x86_64-linux
+
+  - label: 'Make sure materialize function does not limit concurrency'
+    command:
+      - nix-build build.nix -A maintainer-scripts.check-materialization-concurrency -o check-materialization-concurrency.sh
+      - ./check-materialization-concurrency.sh
+
+  - label: 'Make sure non store paths like can be used as src'
+    command:
+      - nix-build build.nix -A maintainer-scripts.check-path-support --argstr compiler-nix-name ghc884 -o check-path-support.sh
+      - ./check-path-support.sh

.gitattributes

@@ -4,3 +4,4 @@
 nix-tools/.plan.nix/*.nix linguist-generated=true
 nix-tools/pkgs.nix linguist-generated=true
 .stack-to-nix.cache linguist-generated=true
+materialized/**/* linguist-generated=true

README.md

@@ -0,0 +1,39 @@
+# `haskell.nix` is infrastructure for building Haskell packages with Nix
+
+[![](https://badge.buildkite.com/d453edcd29bd2f8f3f3b32c9b7d6777a33773d9671c37a6ccc.svg?branch=master)](https://buildkite.com/input-output-hk/haskell-dot-nix)
+[![](https://img.shields.io/buildkite/c8d5a20d3ff0f440f82adb9190b43c16c91e5e47e8adfa867a/master.svg?label=nightly%20updates)](https://buildkite.com/input-output-hk/haskell-dot-nix-nightly-updates)
+
+`haskell.nix` can automatically translate your Cabal or Stack project and
+its dependencies into Nix code.
+
+## Documentation
+
+- [Introduction](https://input-output-hk.github.io/haskell.nix/)
+- [Getting Started](https://input-output-hk.github.io/haskell.nix/tutorials/getting-started/)
+- [Troubleshooting](https://input-output-hk.github.io/haskell.nix/troubleshooting/)
+- Explore the documentation from there to find further topics.
+
+## Help! Something isn't working
+
+The #1 problem that people have when using `haskell.nix` is that they find themselves building GHC.
+This should not happen, but you *must* follow the `haskell.nix` setup instructions properly to avoid it.
+If you find this happening to you, please check that you have followed the 
+[getting started instructions](https://input-output-hk.github.io/haskell.nix/tutorials/getting-started/#setting-up-the-binary-cache) and
+consult the corresponding [troubleshooting section](https://input-output-hk.github.io/haskell.nix/troubleshooting/#why-am-i-building-ghc).
+
+The troubleshooting documentation also contains some help for other common issues.
+Please give it a read before asking on IRC or opening an issue.
+
+If you're still stuck, join the [#haskell.nix](https://www.irccloud.com/invite?channel=%23haskell.nix&hostname=irc.freenode.net&port=6697&ssl=1) channel on [irc.freenode.net](https://freenode.net/), or open an issue.
+
+## Related repos
+
+The `haskell.nix` repository contains the runtime system for building
+Haskell packages in Nix. It depends on other repos, which are:
+
+- [`nix-tools`](https://github.com/input-output-hk/nix-tools) — provides the programs for generating Nix expressions from  Haskell projects.
+
+- [`hackage.nix`](https://github.com/input-output-hk/hackage.nix) — the latest contents of the [Hackage](https://hackage.haskell.org/) databases, converted to Nix expressions.
+
+- [`stackage.nix`](https://github.com/input-output-hk/stackage.nix) — all of the [Stackage](https://www.stackage.org/) snapshots, converted to Nix expressions.
+

README.org

@@ -5,36 +5,48 @@
 let
   haskellNix = (import ./default.nix {});
 in
-{ nixpkgs ? haskellNix.sources.nixpkgs-default
+{ nixpkgs ? haskellNix.sources.nixpkgs
 , nixpkgsArgs ? haskellNix.nixpkgsArgs
 , pkgs ? import nixpkgs nixpkgsArgs
 , ifdLevel ? 1000
+, compiler-nix-name ? throw "No `compiler-nix-name` passed to build.nix"
 }:
 
 let
   haskell = pkgs.haskell-nix;
+  buildHaskell = pkgs.buildPackages.haskell-nix;
+  tool = buildHaskell.tool;
 in rec {
-  tests = import ./test/default.nix { inherit pkgs ifdLevel; };
+  tests = import ./test/default.nix { inherit pkgs ifdLevel compiler-nix-name; };
 
-  tools = pkgs.lib.optionalAttrs (ifdLevel >= 3)
-    (pkgs.recurseIntoAttrs
-      (pkgs.lib.mapAttrs (_: ghc:
-        let
-          tool = name: version: pkgs.buildPackages.haskell-nix.tool name { inherit version ghc; };
-        in pkgs.recurseIntoAttrs {
-            cabal-32 = tool "cabal" "3.2.0.0";
-            ghcide   = tool "ghcide" "object-code";
-          } // pkgs.lib.optionalAttrs (ghc.version == "8.6.5") {
-            cabal-30 = tool "cabal" "3.0.0.0";
-          }) { inherit (pkgs.buildPackages.haskell-nix.compiler) ghc865 ghc883; }));
+  tools = pkgs.lib.optionalAttrs (ifdLevel >= 3) (
+    pkgs.recurseIntoAttrs {
+      cabal-latest = tool compiler-nix-name "cabal" "latest";
+      hls-latest = tool compiler-nix-name "haskell-language-server" "latest";
+      hlint-latest = tool compiler-nix-name "hlint" "latest";
+    }
+  );
 
   # Scripts for keeping Hackage and Stackage up to date, and CI tasks.
   # The dontRecurseIntoAttrs prevents these from building on hydra
   # as not all of them can work in restricted eval mode (as they
   # are not pure).
   maintainer-scripts = pkgs.dontRecurseIntoAttrs {
-    update-hackage = haskell.callPackage ./scripts/update-hackage.nix {};
-    update-stackage = haskell.callPackage ./scripts/update-stackage.nix {};
+    update-hackage = import ./scripts/update-hackage.nix {
+      inherit (pkgs) stdenv writeScript coreutils glibc git
+        openssh nix-prefetch-git gawk bash curl findutils;
+      # Update scripts use the internal nix-tools and cabal-install (compiled with a fixed GHC version)
+      nix-tools = haskell.internal-nix-tools;
+      cabal-install = haskell.internal-cabal-install;
+      inherit (haskell) update-index-state-hashes;
+    };
+    update-stackage = haskell.callPackage ./scripts/update-stackage.nix {
+      inherit (pkgs) stdenv writeScript coreutils glibc git
+        openssh nix-prefetch-git gawk bash curl findutils;
+      # Update scripts use the internal nix-tools and cabal-install (compiled with a fixed GHC version)
+      nix-tools = haskell.internal-nix-tools;
+      cabal-install = haskell.internal-cabal-install;
+    };
     update-pins = haskell.callPackage ./scripts/update-pins.nix {};
     update-docs = pkgs.buildPackages.callPackage ./scripts/update-docs.nix {
       generatedOptions = pkgs.callPackage ./scripts/options-doc.nix { };
@@ -46,8 +58,14 @@ in rec {
     # use)
     check-hydra = pkgs.buildPackages.callPackage ./scripts/check-hydra.nix {};
     check-closure-size = pkgs.buildPackages.callPackage ./scripts/check-closure-size.nix {
-      inherit (haskell) nix-tools;
+      # Includes cabal-install since this is commonly used.
+      nix-tools = pkgs.linkFarm "common-tools" [
+        { name = "nix-tools";     path = haskell.nix-tools.${compiler-nix-name}; }
+        { name = "cabal-install"; path = haskell.cabal-install.${compiler-nix-name}; }
+      ];
     };
+    check-materialization-concurrency = pkgs.buildPackages.callPackage ./scripts/check-materialization-concurrency/check.nix {};
+    check-path-support = pkgs.buildPackages.callPackage ./scripts/check-path-support.nix {};
   };
 
   # These are pure parts of maintainer-script so they can be built by hydra