Merge pull request #194345 from codedownio/binary-cache

 <xi:include href="images/snaptools.section.xml" />

 <xi:include href="images/portableservice.section.xml" />

 <xi:include href="images/makediskimage.section.xml" />

 <xi:include href="images/binarycache.section.xml" />

</chapter>

# pkgs.mkBinaryCache {#sec-pkgs-binary-cache}

`pkgs.mkBinaryCache` is a function for creating Nix flat-file binary caches. Such a cache exists as a directory on disk, and can be used as a Nix substituter by passing `--substituter file:///path/to/cache` to Nix commands.

Nix packages are most commonly shared between machines using [HTTP, SSH, or S3](https://nixos.org/manual/nix/stable/package-management/sharing-packages.html), but a flat-file binary cache can still be useful in some situations. For example, you can copy it directly to another machine, or make it available on a network file system. It can also be a convenient way to make some Nix packages available inside a container via bind-mounting.

Note that this function is meant for advanced use-cases. The more idiomatic way to work with flat-file binary caches is via the [nix-copy-closure](https://nixos.org/manual/nix/stable/command-ref/nix-copy-closure.html) command. You may also want to consider [dockerTools](#sec-pkgs-dockerTools) for your containerization needs.

## Example

The following derivation will construct a flat-file binary cache containing the closure of `hello`.

```nix

mkBinaryCache {

  rootPaths = [hello];

}

```

- `rootPaths` specifies a list of root derivations. The transitive closure of these derivations' outputs will be copied into the cache.

Here's an example of building and using the cache.

Build the cache on one machine, `host1`:

```shellSession

nix-build -E 'with import <nixpkgs> {}; mkBinaryCache { rootPaths = [hello]; }'

```

```shellSession

/nix/store/cc0562q828rnjqjyfj23d5q162gb424g-binary-cache

```

Copy the resulting directory to the other machine, `host2`:

```shellSession

scp result host2:/tmp/hello-cache

```

Substitute the derivation using the flat-file binary cache on the other machine, `host2`:

```shellSession

nix-build -A hello '<nixpkgs>' \

  --option require-sigs false \

  --option trusted-substituters file:///tmp/hello-cache \

  --option substituters file:///tmp/hello-cache

```

```shellSession

/nix/store/gl5a41azbpsadfkfmbilh9yk40dh5dl0-hello-2.12.1

```

  bcachefs = handleTestOn ["x86_64-linux" "aarch64-linux"] ./bcachefs.nix {};

  beanstalkd = handleTest ./beanstalkd.nix {};

  bees = handleTest ./bees.nix {};

  binary-cache = handleTest ./binary-cache.nix {};

  bind = handleTest ./bind.nix {};

  bird = handleTest ./bird.nix {};

  bitcoind = handleTest ./bitcoind.nix {};

import ./make-test-python.nix ({ lib, ... }:

with lib;

{

  name = "binary-cache";

  meta.maintainers = with maintainers; [ thomasjm ];

  nodes.machine =

    { pkgs, ... }: {

      imports = [ ../modules/installer/cd-dvd/channel.nix ];

      environment.systemPackages = with pkgs; [python3];

      system.extraDependencies = with pkgs; [hello.inputDerivation];

      nix.extraOptions = ''

        experimental-features = nix-command

      '';

    };

  testScript = ''

    # Build the cache, then remove it from the store

    cachePath = machine.succeed("nix-build --no-out-link -E 'with import <nixpkgs> {}; mkBinaryCache { rootPaths = [hello]; }'").strip()

    machine.succeed("cp -r %s/. /tmp/cache" % cachePath)

    machine.succeed("nix-store --delete " + cachePath)

    # Sanity test of cache structure

    status, stdout = machine.execute("ls /tmp/cache")

    cache_files = stdout.split()

    assert ("nix-cache-info" in cache_files)

    assert ("nar" in cache_files)

    # Nix store ping should work

    machine.succeed("nix store ping --store file:///tmp/cache")

    # Cache should contain a .narinfo referring to "hello"

    grepLogs = machine.succeed("grep -l 'StorePath: /nix/store/[[:alnum:]]*-hello-.*' /tmp/cache/*.narinfo")

    # Get the store path referenced by the .narinfo

    narInfoFile = grepLogs.strip()

    narInfoContents = machine.succeed("cat " + narInfoFile)

    import re

    match = re.match(r"^StorePath: (/nix/store/[a-z0-9]*-hello-.*)$", narInfoContents, re.MULTILINE)

    if not match: raise Exception("Couldn't find hello store path in cache")

    storePath = match[1]

    # Delete the store path

    machine.succeed("nix-store --delete " + storePath)

    machine.succeed("[ ! -d %s ] || exit 1" % storePath)

    # Should be able to build hello using the cache

    logs = machine.succeed("nix-build -A hello '<nixpkgs>' --option require-sigs false --option trusted-substituters file:///tmp/cache --option substituters file:///tmp/cache 2>&1")

    logLines = logs.split("\n")

    if not "this path will be fetched" in logLines[0]: raise Exception("Unexpected first log line")

    def shouldBe(got, desired):

      if got != desired: raise Exception("Expected '%s' but got '%s'" % (desired, got))

    shouldBe(logLines[1], "  " + storePath)

    shouldBe(logLines[2], "copying path '%s' from 'file:///tmp/cache'..." % storePath)

    shouldBe(logLines[3], storePath)

    # Store path should exist in the store now

    machine.succeed("[ -d %s ] || exit 1" % storePath)

  '';

})

{ stdenv, buildPackages }:

# This function is for creating a flat-file binary cache, i.e. the kind created by

# nix copy --to file:///some/path and usable as a substituter (with the file:// prefix).

# For example, in the Nixpkgs repo:

# nix-build -E 'with import ./. {}; mkBinaryCache { rootPaths = [hello]; }'

{ name ? "binary-cache"

, rootPaths

}:

stdenv.mkDerivation {

  inherit name;

  __structuredAttrs = true;

  exportReferencesGraph.closure = rootPaths;

  preferLocalBuild = true;

  PATH = "${buildPackages.coreutils}/bin:${buildPackages.jq}/bin:${buildPackages.python3}/bin:${buildPackages.nix}/bin:${buildPackages.xz}/bin";

  builder = builtins.toFile "builder" ''

    . .attrs.sh

    export out=''${outputs[out]}

    mkdir $out

    mkdir $out/nar

    python ${./make-binary-cache.py}

    # These directories must exist, or Nix might try to create them in LocalBinaryCacheStore::init(),

    # which fails if mounted read-only

    mkdir $out/realisations

    mkdir $out/debuginfo

    mkdir $out/log

  '';

}