Merge remote-tracking branch 'origin/master' into support-cabal-doctest

Author: hamishmack

README.org

@@ -20,13 +20,26 @@ For the documentation, see https://input-output-hk.github.io/haskell.nix/.
 For =cabal.project= project add a =default.nix=:
 
 #+begin_src sh
-{ pkgs ? import <nixpkgs> (import (builtins.fetchTarball https://github.com/input-output-hk/haskell.nix/archive/master.tar.gz))
+let 
+  # Fetch the latest haskell.nix and import its default.nix 
+  haskellNix = import (builtins.fetchTarball https://github.com/input-output-hk/haskell.nix/archive/master.tar.gz) {};
+  # haskell.nix provides access to the nixpkgs pins which are used by our CI, hence
+  # you will be more likely to get cache hits when using these. 
+  # But you can also just use your own, e.g. '<nixpkgs>'
+  nixpkgsSrc = haskellNix.sources.nixpkgs-1909;
+  # haskell.nix provides some arguments to be passed to nixpkgs, including some patches
+  # and also the haskell.nix functionality itself as an overlay.
+  nixpkgsArgs = haskellNix.nixpkgsArgs;
+in
+{ pkgs ? import nixpkgsSrc nixpkgsArgs
 , haskellCompiler ? "ghc865"
 }:
-  pkgs.haskell-nix.cabalProject {
-    src = pkgs.haskell-nix.haskellLib.cleanGit { src = ./.; };
-    ghc = pkgs.buildPackages.pkgs.haskell-nix.compiler.${haskellCompiler};
-  }
+# 'cabalProject' generates a package set based on a cabal.project (and the corresponding .cabal files)
+pkgs.haskell-nix.cabalProject {
+  # 'cleanGit' cleans a source directory based on the files known by git
+  src = pkgs.haskell-nix.haskellLib.cleanGit { src = ./.; };
+  ghc = pkgs.buildPackages.pkgs.haskell-nix.compiler.${haskellCompiler};
+}
 #+end_src
 
 Note that you'll need to add a comment specifying the expected sha256
@@ -45,11 +58,16 @@ source-repository-package
 For a =stack.yaml= project add a =default.nix=:
 
 #+begin_src sh
-{ pkgs ? import <nixpkgs> (import (builtins.fetchTarball https://github.com/input-output-hk/haskell.nix/archive/master.tar.gz))
+let 
+  haskellNix = import (builtins.fetchTarball https://github.com/input-output-hk/haskell.nix/archive/master.tar.gz) {};
+  nixpkgsSrc = haskellNix.sources.nixpkgs-1909;
+  nixpkgsArgs = haskellNix.nixpkgsArgs;
+in
+{ pkgs ? import nixpkgsSrc nixpkgsArgs
 }:
-  pkgs.haskell-nix.stackProject {
-    src = pkgs.haskell-nix.haskellLib.cleanGit { src = ./.; };
-  }
+pkgs.haskell-nix.stackProject {
+  src = pkgs.haskell-nix.haskellLib.cleanGit { src = ./.; };
+}
 #+end_src
 
 To build the library component of a package in the project run:

build.nix

@@ -2,26 +2,19 @@
 #
 # It is separate from default.nix because that file is the public API
 # of Haskell.nix, which shouldn't have tests, etc.
-
-{ nixpkgs ? ./nixpkgs
-# Provide args to the nixpkgs instantiation.
-, system ? builtins.currentSystem
-, crossSystem ? null
-, config ? {}
-, nixpkgsArgs ? { inherit system crossSystem; }
+let
+  haskellNix = (import ./default.nix {});
+in
+{ nixpkgs ? haskellNix.sources.nixpkgs-default
+, nixpkgsArgs ? haskellNix.nixpkgsArgs
+, pkgs ? import nixpkgs nixpkgsArgs
 , ifdLevel ? 1000
 }:
 
 let
-  haskellNixArgs = import ./default.nix;
-  pkgs = import nixpkgs ({
-      config = haskellNixArgs.config // config;
-      inherit (haskellNixArgs) overlays;
-    } // nixpkgsArgs);
   haskell = pkgs.haskell-nix;
-
 in rec {
-  tests = import ./test/default.nix { inherit nixpkgs nixpkgsArgs ifdLevel; };
+  tests = import ./test/default.nix { inherit pkgs ifdLevel; };
 
   # Scripts for keeping Hackage and Stackage up to date, and CI tasks.
   # The dontRecurseIntoAttrs prevents these from building on hydra
@@ -36,7 +29,7 @@ in rec {
         # nixpkgs 19.09 changes "Option has no description" from an
         # error into a warning. That is quite helpful when hardly any
         # of our options are documented, thanks @oxij.
-        pkgs = import ./nixpkgs { nixpkgs-pin = "release-19.09"; };
+        pkgs = import (import ./nixpkgs/default.nix).nixpkgs-1909 {};
       };
     };
     # Because this is going to be used to test caching on hydra, it must not

builder/default.nix

@@ -40,7 +40,7 @@ let
 
   hoogleLocal = let
     # Use the latest default nixpkgs hoogle.nix, as the 19.03 one does not work with cross compilers
-    nixpkgsHoogleLocal = import ((import ../nixpkgs {}).path + /pkgs/development/haskell-modules/hoogle.nix);
+    nixpkgsHoogleLocal = import ((import (import ../nixpkgs).nixpkgs-default {}).path + /pkgs/development/haskell-modules/hoogle.nix);
   in { packages ? [], hoogle ? pkgs.buildPackages.haskell-nix.haskellPackages.hoogle.components.exes.hoogle }:
     haskellLib.weakCallPackage pkgs nixpkgsHoogleLocal {
       # For musl we can use haddock from the buildGHC

builder/make-config-files.nix

@@ -170,7 +170,7 @@ in { identifier, component, fullName, flags ? {} }:
       done | sed 's|''${pkgroot}/../../../|/nix/store/|' | sort -u
     )
     for d in $dirsToLink; do
-      ln -f -s "$d/"*.dylib $dynamicLinksDir
+      ln -f -s "$d/"*.{a,dylib,so} $dynamicLinksDir
     done
     # Edit the local package DB to reference the links directory.
     for f in "$out/${packageCfgDir}/"*.conf; do

ci-lib.nix

@@ -0,0 +1,45 @@
+let
+  # Generic nixpkgs, use *only* for lib functions that are stable across versions
+  pkgs = import (import ./nixpkgs/default.nix).nixpkgs-default {};
+  lib = pkgs.lib;
+in rec {
+  inherit (import ./dimension.nix) dimension;
+
+  # A filter for removing packages that aren't supported on the current platform
+  # according to 'meta.platforms'.
+  platformFilterGeneric = pkgs: system:
+    # This needs to use the correct nixpkgs version so all the systems line up
+    let lib = pkgs.lib;
+        platform = lib.systems.elaborate { inherit system; };
+    # Can't just default to [] for platforms, since no meta.platforms
+    # means "all platforms" not "no platforms"
+    in drv : if drv ? meta && drv.meta ? platforms then
+      lib.any (lib.meta.platformMatch platform) drv.meta.platforms
+    else true;
+
+  # Hydra doesn't like these attributes hanging around in "jobsets": it thinks they're jobs!
+  stripAttrsForHydra = filterAttrsOnlyRecursive (n: _: n != "recurseForDerivations" && n != "dimension");
+
+  # Keep derivations and attrsets with 'recurseForDerivations'. This ensures that we match the
+  # derivations that Hercules will see, and prevents Hydra from trying to pick up all sorts of bad stuff
+  # (like attrsets that contain themselves!).
+  filterDerivations = filterAttrsOnlyRecursive (n: attrs: lib.isDerivation attrs || attrs.recurseForDerivations or false);
+
+  # A version of 'filterAttrsRecursive' that doesn't recurse into derivations. This prevents us from going into an infinite
+  # loop with the 'out' attribute on derivations.
+  # TODO: Surely this shouldn't be necessary. I think normal 'filterAttrsRecursive' will effectively cause infinite loops
+  # if you keep derivations and your predicate forces the value of the attribute, as this then triggers a loop on the
+  # 'out' attribute. Weird.
+  filterAttrsOnlyRecursive = pred: set:
+    lib.listToAttrs (
+      lib.concatMap (name:
+        let v = set.${name}; in
+        if pred name v then [
+          (lib.nameValuePair name (
+            if builtins.isAttrs v && !lib.isDerivation v then filterAttrsOnlyRecursive pred v
+            else v
+          ))
+        ] else []
+      ) (builtins.attrNames set)
+    );
+}

ci.nix

@@ -1,44 +1,65 @@
-# these are patched nixpkgs that include the following PRs:
-# - https://github.com/NixOS/nixpkgs/pull/71216
-# - https://github.com/NixOS/nixpkgs/pull/68398
-
+# 'supportedSystems' restricts the set of systems that we will evaluate for. Useful when you're evaluting
+# on a machine with e.g. no way to build the Darwin IFDs you need! 
+{ supportedSystems ? [ "x86_64-linux" "x86_64-darwin" ]
+, ifdLevel ? 3
+# Whether or not we are evaluating in restricted mode. This is true in Hydra, but not in Hercules.
+, restrictEval ? false }:
 let
-  inherit (import ./dimension.nix) dimension;
+  inherit (import ./ci-lib.nix) dimension platformFilterGeneric filterAttrsOnlyRecursive;
+  inherit (import ./default.nix {}) sources nixpkgsArgs;
   nixpkgsVersions = {
-#   "release-18.09" = builtins.fetchTarball "https://github.com/input-output-hk/nixpkgs/archive/7e4dcacbf066a8e2d12693a9de1fb30c77081c5d.tar.gz";
-    "release-19.03" = builtins.fetchTarball "https://github.com/input-output-hk/nixpkgs/archive/a8f81dc037a5977414a356dd068f2621b3c89b60.tar.gz";
-    # "release-19.09" = builtins.fetchTarball "https://github.com/input-output-hk/nixpkgs/archive/3d623a406cec9052ae0a16a79ce3ce9de11236bb.tar.gz";
+    "R1903" = "nixpkgs-1903";
+    "R1909" = "nixpkgs-1909";
   };
-  systems = {
-    "x86_64-linux" = {};
+  systems = nixpkgs: nixpkgs.lib.filterAttrs (_: v: builtins.elem v supportedSystems) {
+    # I wanted to take these from 'lib.systems.examples', but apparently there isn't one for linux!
+    linux = "x86_64-linux";
+    darwin = "x86_64-darwin";
   };
-  crossSystems = {
-    "x86_64-pc-mingw32" = {};
+  crossSystems = nixpkgsName: nixpkgs: system:
+    # We need to use the actual nixpkgs version we're working with here, since the values
+    # of 'lib.systems.examples' are not understood between all versions
+    let lib = nixpkgs.lib;
+    in lib.optionalAttrs (system == "x86_64-linux" || nixpkgsName == "R1903") {
+    # Windows cross compilation is currently broken on macOS for nixpkgs 19.09 (works on 19.03)
+    inherit (lib.systems.examples) mingwW64;
+  } // lib.optionalAttrs (system == "x86_64-linux") {
+    # Musl cross only works on linux
+    # aarch64 cross only works on linux
+    inherit (lib.systems.examples) musl64 aarch64-multiplatform;
   };
-  haskellNixArgs = import ./.;
 in
-dimension "Nixpkgs version" nixpkgsVersions (nixpkgsName: nixpkgsSrc:
-  dimension "System" systems (system: _:
-    # Native builds
-    # TODO: can we merge this into the general case by picking an appropriate "cross system" to mean native?
-    let pkgs = import nixpkgsSrc (haskellNixArgs // { inherit system; });
-    in pkgs.recurseIntoAttrs {
-      hello = (pkgs.haskell-nix.hackage-package { name = "hello"; version = "1.0.0.2";}).components.exes.hello;
-      iserv-proxy = pkgs.ghc-extra-packages.ghc865.iserv-proxy.components.exes.iserv-proxy;
-      ghc = pkgs.recurseIntoAttrs pkgs.haskell-nix.compiler;
-      tests = pkgs.lib.optionalAttrs (system == "x86_64-linux") (import ./test { inherit pkgs; });
+dimension "Nixpkgs version" nixpkgsVersions (nixpkgsName: nixpkgs-pin:
+  let pinnedNixpkgsSrc = sources.${nixpkgs-pin};
+      # We need this for generic nixpkgs stuff at the right version
+      genericPkgs = import pinnedNixpkgsSrc {};
+  in dimension "System" (systems genericPkgs) (systemName: system:
+    let pkgs = import pinnedNixpkgsSrc (nixpkgsArgs // { inherit system; });
+        build = import ./build.nix { inherit pkgs ifdLevel; };
+        platformFilter = platformFilterGeneric pkgs system;
+    in filterAttrsOnlyRecursive (_: v: platformFilter v) {
+      # Native builds
+      # TODO: can we merge this into the general case by picking an appropriate "cross system" to mean native?
+      native = pkgs.recurseIntoAttrs {
+        inherit (build) tests maintainer-scripts maintainer-script-cache;
+        hello = (pkgs.haskell-nix.hackage-package { name = "hello"; version = "1.0.0.2"; }).components.exes.hello;
+        iserv-proxy = pkgs.ghc-extra-packages.ghc865.iserv-proxy.components.exes.iserv-proxy;
+        ghc = pkgs.recurseIntoAttrs pkgs.haskell-nix.compiler;
+      };
     }
     //
-    dimension "Cross system" crossSystems (crossSystem: _:
+    dimension "Cross system" (crossSystems nixpkgsName genericPkgs system) (crossSystemName: crossSystem:
       # Cross builds
-      let pkgs = import nixpkgsSrc (haskellNixArgs // { inherit system; crossSystem.config = crossSystem; });
+      let pkgs = import pinnedNixpkgsSrc (nixpkgsArgs // { inherit system crossSystem; });
+          build = import ./build.nix { inherit pkgs ifdLevel; };
       in pkgs.recurseIntoAttrs {
-        hello = (pkgs.haskell-nix.hackage-package { name = "hello"; version = "1.0.0.2";}).components.exes.hello;
-
+        hello = (pkgs.haskell-nix.hackage-package { name = "hello"; version = "1.0.0.2"; }).components.exes.hello;
         iserv-proxy = pkgs.ghc-extra-packages.ghc865.iserv-proxy.components.exes.iserv-proxy;
-
         remote-iserv = pkgs.ghc-extra-packages.ghc865.remote-iserv.components.exes.remote-iserv;
       }
+      //
+      # Tests are broken on aarch64 cross https://github.com/input-output-hk/haskell.nix/issues/513
+      pkgs.lib.optionalAttrs (crossSystemName != "aarch64-multiplatform") build.tests
     )
   )
 )

compiler/ghc/default.nix

@@ -73,9 +73,12 @@ let
   inherit (bootPkgs) ghc;
 
   # TODO check if this posible fix for segfaults works or not.
-  libffiStaticEnabled = if libffi == null || !stdenv.targetPlatform.isMusl
-    then libffi
-    else targetPackages.libffi.overrideAttrs (old: { dontDisableStatic = true; });
+  targetLibffi =
+    if stdenv.targetPlatform.isMusl
+    then targetPackages.libffi.overrideAttrs (old: { dontDisableStatic = true; })
+    else if targetPlatform != hostPlatform
+    then targetPackages.libffi
+    else libffi;
 
   # TODO(@Ericson2314) Make unconditional
   targetPrefix = stdenv.lib.optionalString
@@ -116,7 +119,7 @@ let
 
   # Splicer will pull out correct variations
   libDeps = platform: stdenv.lib.optional enableTerminfo [ ncurses ]
-    ++ [libffiStaticEnabled]
+    ++ [targetLibffi]
     ++ stdenv.lib.optional (!enableIntegerSimple) gmp
     ++ stdenv.lib.optional (platform.libc != "glibc" && !targetPlatform.isWindows) libiconv;
 
@@ -208,7 +211,7 @@ in let configured-src = stdenv.mkDerivation (rec {
         configureFlags = [
             "--datadir=$doc/share/doc/ghc"
             "--with-curses-includes=${ncurses.dev}/include" "--with-curses-libraries=${ncurses.out}/lib"
-        ] ++ stdenv.lib.optionals (libffiStaticEnabled != null) ["--with-system-libffi" "--with-ffi-includes=${libffiStaticEnabled.dev}/include" "--with-ffi-libraries=${libffiStaticEnabled.out}/lib"
+        ] ++ stdenv.lib.optionals (targetLibffi != null) ["--with-system-libffi" "--with-ffi-includes=${targetLibffi.dev}/include" "--with-ffi-libraries=${targetLibffi.out}/lib"
         ] ++ stdenv.lib.optional (!enableIntegerSimple) [
             "--with-gmp-includes=${targetPackages.gmp.dev}/include" "--with-gmp-libraries=${targetPackages.gmp.out}/lib"
         ] ++ stdenv.lib.optional (targetPlatform == hostPlatform && hostPlatform.libc != "glibc" && !targetPlatform.isWindows) [

default.nix

@@ -1,5 +1,24 @@
-# This default.nix is designed to be passed directly nixpkgs with something like:
-#   import <nixpkgs> (import (builtins.fetchTarball https://github.com/input-output-hk/haskell.nix/archive/master.tar.gz))
-{ config   = import ./config.nix;
-  overlays = import ./overlays;
-}
+let haskellNix = rec {
+    sources = {
+      inherit (import ./nixpkgs/default.nix) nixpkgs-1903 nixpkgs-1909 nixpkgs-default;
+    };
+
+    config   = import ./config.nix;
+    overlays = import ./overlays;
+    nixpkgsArgs = { inherit overlays config; };
+  };
+
+  haskellNixV1 = haskellNix.nixpkgsArgs;
+  haskellNixV2 = haskellNix;
+
+  v1DeprecationMessage = "Version 1 is deprecated: use version 2 (nixpkgs arguments are available as the `nixpkgsArgs` attribute of version 2)";
+# If no arguments, then you get V1
+# I'd like to make importing directly issue a warning, but I couldn't figure out a way to make it happen
+in haskellNixV1 // {
+  __functor = _: { version ? 2 }:
+    if version == 1
+    then builtins.trace v1DeprecationMessage haskellNixV1
+    else if version == 2
+    then haskellNixV2
+    else builtins.throw ("haskell.nix: unknown version: " + (builtins.toString version));
+}

docs/user-guide.md

@@ -7,22 +7,22 @@
 To build a package, say [lens][], from a stackage snapshot, say [lts-13.28][],
 you could run
 ```bash
-nix build '(with import <nixpkgs> (import ./.); haskell-nix.snapshots."lts-13.28").lens.components.library'
+nix build '(with import <nixpkgs> (import ./. {}).nixpkgsArgs; haskell-nix.snapshots."lts-13.28").lens.components.library'
 ```
 which would build the [lens][] library component from the lens package as fixed
 by the [lts-13.28][] stackage snapshot.
 
 To build any package from hackage, say [lens][], in version, say 4.17.1, you
 could run
 ```bash
-nix build '(with import <nixpkgs> (import ./.); (haskell-nix.hackage-package { name = "lens"; version = "4.17.1"; })).components.library'
+nix build '(with import <nixpkgs> (import ./. {}).nixpkgsArgs; (haskell-nix.hackage-package { name = "lens"; version = "4.17.1"; })).components.library'
 ```
 which would build the [lens][] library component from the [lens-4.17.1][] package
 from hackage.  The dependencies would be solved against the most recent 
 [hackage-index-state][] that comes via the [hackage.nix][] pin with your
 [haskell.nix][] checkout.  A specific one can be specified as well:
 ```bash
-nix build '(with import <nixpkgs> (import ./.); (haskell-nix.hackage-package { name = "lens"; version = "4.17.1"; index-state = "2019-07-14T00:00:00Z"; })).components.library'
+nix build '(with import <nixpkgs> (import ./. {}).nixpkgsArgs; (haskell-nix.hackage-package { name = "lens"; version = "4.17.1"; index-state = "2019-07-14T00:00:00Z"; })).components.library'
 ```
 which would use the hackage index as of `2019-07-14T00:00:00Z` to produce a build plan
 for the [lens-4.17.1][] package.
@@ -49,7 +49,7 @@ produce derivations that we can `nix build`.
 To build the latest `nix-tools` and store the result at `./nt`, run:
 
 ```bash
-nix build '(with import <nixpkgs> (import (builtins.fetchTarball https://github.com/input-output-hk/haskell.nix/archive/master.tar.gz)); haskell-nix.nix-tools)' --out-link nt
+nix build '(with import <nixpkgs> (import (builtins.fetchTarball https://github.com/input-output-hk/haskell.nix/archive/master.tar.gz) {}).nixpkgsArgs; haskell-nix.nix-tools)' --out-link nt
 ```
 
 If you would like to then install `nix-tools` into your profile, run:
@@ -85,15 +85,15 @@ The easiest way to get a hold of [Haskell.nix][] is with
 [`fetchTarball`](https://nixos.org/nix/manual/#ssec-builtins).
 
 ```nix
-import <nixpkgs> (import (builtins.fetchTarball https://github.com/input-output-hk/haskell.nix/archive/master.tar.gz))
+import <nixpkgs> (import (builtins.fetchTarball https://github.com/input-output-hk/haskell.nix/archive/master.tar.gz) {}).nixpkgsArgs
 ```
 
 ### Using your cabal.project file
 
 If your project has a `cabal.project` you can add a `default.nix` like this:
 
 ```nix
-{ pkgs ? import <nixpkgs> (import (builtins.fetchTarball https://github.com/input-output-hk/haskell.nix/archive/master.tar.gz))
+{ pkgs ? import <nixpkgs> (import (builtins.fetchTarball https://github.com/input-output-hk/haskell.nix/archive/master.tar.gz) {}).nixpkgsArgs
 , haskellCompiler ? "ghc865"
 }:
   pkgs.haskell-nix.cabalProject {
@@ -173,7 +173,7 @@ or one of the constraints in your local `.cabal` files).
 If your project has a `stack.yaml` you can add a `default.nix` like this:
 
 ```nix
-{ pkgs ? import <nixpkgs> (import (builtins.fetchTarball https://github.com/input-output-hk/haskell.nix/archive/master.tar.gz))
+{ pkgs ? import <nixpkgs> (import (builtins.fetchTarball https://github.com/input-output-hk/haskell.nix/archive/master.tar.gz) {}).nixpkgsArgs
 }:
   pkgs.haskell-nix.stackProject {
     src = pkgs.haskell-nix.haskellLib.cleanGit { src = ./.; };
@@ -269,7 +269,7 @@ these Git repositories correspond to the actual Hackage and Stackage.
       stackageSourceJSON = ./stackage-src.json;
     };
   })]; })
-, haskellNixArgs ? import (builtins.fetchTarball https://github.com/input-output-hk/haskell.nix/archive/master.tar.gz)
+, haskellNixArgs ? (import (builtins.fetchTarball https://github.com/input-output-hk/haskell.nix/archive/master.tar.gz) {}).nixpkgsArgs
 }:
   pkgs
 ```
@@ -283,7 +283,7 @@ attrsets and try examples.
 # example.nix
 { nixpkgs ? <nixpkgs> }:
 rec {
-  haskell = import nixpkgs (import (builtins.fetchTarball https://github.com/input-output-hk/haskell.nix/archive/master.tar.gz));
+  haskell = import nixpkgs (import (builtins.fetchTarball https://github.com/input-output-hk/haskell.nix/archive/master.tar.gz) {}).nixpkgsArgs;
   pkgNames = haskell.pkgs.lib.attrNames haskell.haskell-nix.snapshots."lts-13.18";
 }
 ```