all: refactor to follow RFC140

callUnitRoot.nix

@@ -1,15 +0,0 @@
-{ pkgs
-, lib ? pkgs.lib
-, unitDir ? "unit"
-, packageFun ? "package.nix"
-, root ? "${./pkgs}/${unitDir}"
-}:
-let
-  shards = lib.attrNames (builtins.readDir root);
-  namesForShard = shard: lib.mapAttrs'
-    (name: _: { inherit name; value = root + "/${shard}/${name}"; })
-    (builtins.readDir (root + "/${shard}"));
-  namesToPath = lib.foldl' lib.recursiveUpdate { } (map namesForShard shards);
-  units = lib.mapAttrs (_: path: pkgs.callPackage (path + "/${packageFun}") { }) namesToPath;
-in
-units

default.nix

@@ -8,10 +8,10 @@
 
 { pkgs ? import <nixpkgs> { } }:
 let
-  legacy = import ./pkgs/top-level/all-packages.nix { inherit pkgs; };
+  system = pkgs.stdenv.hostPlatform.system;
-  units = import ./callUnitRoot.nix { inherit pkgs; root = ./pkgs/unit; };
+  packages = import ./pkgs/top-level { localSystem = system; inherit pkgs; };
 in
-legacy // units // {
+packages // {
   # The `lib`, `modules`, and `overlay` names are special
   lib = import ./lib { inherit pkgs; }; # functions
   modules = import ./modules; # NixOS modules

flake.nix

@@ -15,16 +15,7 @@
           pkgs = import nixpkgs { inherit system; };
         in
         {
-          # nixos/rfcs#140
+          packages = import ./pkgs/top-level { localSystem = system; inherit pkgs; };
-          # Only produces the package set of the proposed functionality.
-          # Unstable names are variables.
-          packages =
-            let
-              legacyPackages = import ./pkgs/top-level/all-packages.nix { inherit pkgs; };
-              unitPackages = import ./callUnitRoot.nix { inherit pkgs; };
-              onlyAvailable = lib.filterAttrs (_: drv: builtins.elem system (drv.meta.platforms or [ ]));
-            in
-            onlyAvailable (legacyPackages // unitPackages);
 
           checks = {
             nixpkgs-fmt = pkgs.writeShellScriptBin "nixpkgs-fmt-check" ''

pkgs/top-level/by-name-overlay.nix

@@ -0,0 +1,56 @@
+# This file turns the pkgs/by-name directory (see its README.md for more info)
+# into an overlay that adds all the defined packages.
+#
+# No validity checks are done here, instead this file is optimised for
+# performance, and validity checks are done by CI on PRs.
+#
+# This file is based on Nixpkgs' `pkgs/top-level/by-name-overlay.nix` in order
+# to utilise the same infrastructure and layout, with some adjustments to fit
+# our derivative project.
+
+{ lib
+, pkgs
+}:
+
+# Type: Path -> Overlay
+baseDirectory:
+let
+  inherit (builtins)
+    readDir
+    ;
+
+  inherit (lib.attrsets)
+    mapAttrs
+    mapAttrsToList
+    mergeAttrsList
+    ;
+
+  # Package files for a single shard
+  # Type: String -> String -> AttrsOf Path
+  namesForShard = shard: type:
+    if type != "directory" then
+    # Ignore all non-directories. Technically only README.md is allowed as a file in the base directory, so we could alternatively:
+    # - Assume that README.md is the only file and change the condition to `shard == "README.md"` for a minor performance improvement.
+    #   This would however cause very poor error messages if there's other files.
+    # - Ensure that README.md is the only file, throwing a better error message if that's not the case.
+    #   However this would make for a poor code architecture, because one type of error would have to be duplicated in the validity checks and here.
+    # Additionally in either of those alternatives, we would have to duplicate the hardcoding of "README.md"
+      { }
+    else
+      mapAttrs
+        (name: _: baseDirectory + "/${shard}/${name}/package.nix")
+        (readDir (baseDirectory + "/${shard}"));
+
+  # The attribute set mapping names to the package files defining them
+  # This is defined up here in order to allow reuse of the value (it's kind of expensive to compute)
+  # if the overlay has to be applied multiple times
+  packageFiles = mergeAttrsList (mapAttrsToList namesForShard (readDir baseDirectory));
+in
+# TODO: Consider optimising this using `builtins.deepSeq packageFiles`,
+  # which could free up the above thunks and reduce GC times.
+  # Currently this would be hard to measure until we have more packages
+  # and ideally https://github.com/NixOS/nix/pull/8895
+_self: _super:
+mapAttrs
+  (_name: file: pkgs.callPackage file { })
+  packageFiles

pkgs/top-level/default.nix

@@ -0,0 +1,19 @@
+# Composes the packages collection.
+
+{
+  # The system packages will be build and used on.
+  localSystem
+  # Nixpkgs
+, pkgs
+  # Nixpkgs lib
+, lib ? pkgs.lib
+}:
+let
+  allPackages = import ./stage.nix {
+    inherit lib pkgs;
+  };
+
+  available = lib.filterAttrs
+    (_: drv: builtins.elem localSystem (drv.meta.platforms or [ ]));
+in
+available allPackages

pkgs/top-level/stage.nix

@@ -0,0 +1,24 @@
+# Composes a single bootstrapping of the package collection. The result is a set
+# of all the packages for some particular platform.
+
+{ lib
+, pkgs
+}:
+let
+
+  # An overlay to auto-call packages in .../by-name.
+  autoCalledPackages =
+    import ./by-name-overlay.nix { inherit pkgs lib; } ../by-name;
+
+  allPackages = _self: _super:
+    import ./all-packages.nix { inherit pkgs; };
+
+  toFix = (lib.flip lib.composeManyExtensions) (_self: { }) [
+    autoCalledPackages
+    allPackages
+  ];
+
+in
+
+# Return the complete set of packages.
+lib.fix toFix