rustPlatform.fetchCargoVendor: init (#349360)

```

Exception: If the application has cargo `git` dependencies, the `cargoHash`

approach will not work, and you will need to copy the `Cargo.lock` file of the application

to nixpkgs and continue with the next section for specifying the options of the `cargoLock`

section.

approach will not work by default. In this case, you can set `useFetchCargoVendor = true`

to use an improved fetcher that supports handling `git` dependencies.

```nix

{

  useFetchCargoVendor = true;

  cargoHash = "sha256-RqPVFovDaD2rW31HyETJfQ0qVwFxoGEvqkIgag3H6KU=";

}

```

If this method still does not work, you can resort to copying the `Cargo.lock` file into nixpkgs

and importing it as described in the [next section](#importing-a-cargo.lock-file).

Both types of hashes are permitted when contributing to nixpkgs. The

Cargo hash is obtained by inserting a fake checksum into the

  the `Cargo.lock`/`Cargo.toml` files need to be patched before

  vendoring.

In case the lockfile contains cargo `git` dependencies, you can use

`fetchCargoVendor` instead.

```nix

{

  cargoDeps = rustPlatform.fetchCargoVendor {

    inherit src;

    hash = "sha256-RqPVFovDaD2rW31HyETJfQ0qVwFxoGEvqkIgag3H6KU=";

  };

}

```

If a `Cargo.lock` file is available, you can alternatively use the

`importCargoLock` function. In contrast to `fetchCargoTarball`, this

function does not require a hash (unless git dependencies are used)

{ lib

, importCargoLock

, fetchCargoTarball

, fetchCargoVendor

, stdenv

, callPackage

, cargoBuildHook

, cargoDepsHook ? ""

, buildType ? "release"

, meta ? {}

, useFetchCargoVendor ? false

, cargoLock ? null

, cargoVendorDir ? null

, checkType ? buildType

  cargoDeps =

    if cargoVendorDir != null then null

    else if cargoLock != null then importCargoLock cargoLock

    else if useFetchCargoVendor then (fetchCargoVendor {

      inherit src srcs sourceRoot preUnpack unpackPhase postUnpack;

      name = cargoDepsName;

      patches = cargoPatches;

      hash = args.cargoHash;

    } // depsExtraArgs)

    else fetchCargoTarball ({

      inherit src srcs sourceRoot preUnpack unpackPhase postUnpack cargoUpdateHook;

      name = cargoDepsName;

          echo

          echo "ERROR: The Cargo.lock contains git dependencies"

          echo

          echo "This is currently not supported in the fixed-output derivation fetcher."

          echo "Use cargoLock.lockFile / importCargoLock instead."

          echo "This is not supported in the default fixed-output derivation fetcher."

          echo "Set \`useFetchCargoVendor = true\` / use fetchCargoVendor"

          echo "or use cargoLock.lockFile / importCargoLock instead."

          echo

          exit 1

import functools

import hashlib

import json

import multiprocessing as mp

import re

import shutil

import subprocess

import sys

import tomllib

from pathlib import Path

from typing import Any, TypedDict, cast

import requests

eprint = functools.partial(print, file=sys.stderr)

def load_toml(path: Path) -> dict[str, Any]:

    with open(path, "rb") as f:

        return tomllib.load(f)

def download_file_with_checksum(url: str, destination_path: Path) -> str:

    sha256_hash = hashlib.sha256()

    with requests.get(url, stream=True) as response:

        if not response.ok:

            raise Exception(f"Failed to fetch file from {url}. Status code: {response.status_code}")

        with open(destination_path, "wb") as file:

            for chunk in response.iter_content(1024):  # Download in chunks

                if chunk:  # Filter out keep-alive chunks

                    file.write(chunk)

                    sha256_hash.update(chunk)

    # Compute the final checksum

    checksum = sha256_hash.hexdigest()

    return checksum

def get_download_url_for_tarball(pkg: dict[str, Any]) -> str:

    # TODO: support other registries

    #       maybe fetch config.json from the registry root and get the dl key

    #       See: https://doc.rust-lang.org/cargo/reference/registry-index.html#index-configuration

    if pkg["source"] != "registry+https://github.com/rust-lang/crates.io-index":

        raise Exception("Only the default crates.io registry is supported.")

    return f"https://crates.io/api/v1/crates/{pkg["name"]}/{pkg["version"]}/download"

def download_tarball(pkg: dict[str, Any], out_dir: Path) -> None:

    url = get_download_url_for_tarball(pkg)

    filename = f"{pkg["name"]}-{pkg["version"]}.tar.gz"

    # TODO: allow legacy checksum specification, see importCargoLock for example

    #       also, don't forget about the other usage of the checksum

    expected_checksum = pkg["checksum"]

    tarball_out_dir = out_dir / "tarballs" / filename

    eprint(f"Fetching {url} -> tarballs/{filename}")

    calculated_checksum = download_file_with_checksum(url, tarball_out_dir)

    if calculated_checksum != expected_checksum:

        raise Exception(f"Hash mismatch! File fetched from {url} had checksum {calculated_checksum}, expected {expected_checksum}.")

def download_git_tree(url: str, git_sha_rev: str, out_dir: Path) -> None:

    tree_out_dir = out_dir / "git" / git_sha_rev

    eprint(f"Fetching {url}#{git_sha_rev} -> git/{git_sha_rev}")

    cmd = ["nix-prefetch-git", "--builder", "--quiet", "--url", url, "--rev", git_sha_rev, "--out", str(tree_out_dir)]

    subprocess.check_output(cmd)

GIT_SOURCE_REGEX = re.compile("git\\+(?P<url>[^?]+)(\\?(?P<type>rev|tag|branch)=(?P<value>.*))?#(?P<git_sha_rev>.*)")

class GitSourceInfo(TypedDict):

    url: str

    type: str | None

    value: str | None

    git_sha_rev: str

def parse_git_source(source: str) -> GitSourceInfo:

    match = GIT_SOURCE_REGEX.match(source)

    if match is None:

        raise Exception(f"Unable to process git source: {source}.")

    return cast(GitSourceInfo, match.groupdict(default=None))

def create_vendor_staging(lockfile_path: Path, out_dir: Path) -> None:

    cargo_toml = load_toml(lockfile_path)

    git_packages: list[dict[str, Any]] = []

    registry_packages: list[dict[str, Any]] = []

    for pkg in cargo_toml["package"]:

        # ignore local dependenices

        if "source" not in pkg.keys():

            eprint(f"Skipping local dependency: {pkg["name"]}")

            continue

        source = pkg["source"]

        if source.startswith("git+"):

            git_packages.append(pkg)

        elif source.startswith("registry+"):

            registry_packages.append(pkg)

        else:

            raise Exception(f"Can't process source: {source}.")

    git_sha_rev_to_url: dict[str, str] = {}

    for pkg in git_packages:

        source_info = parse_git_source(pkg["source"])

        git_sha_rev_to_url[source_info["git_sha_rev"]] = source_info["url"]

    out_dir.mkdir(exist_ok=True)

    shutil.copy(lockfile_path, out_dir / "Cargo.lock")

    # create a pool with at most 10 concurrent jobs

    with mp.Pool(min(10, mp.cpu_count())) as pool:

        if len(git_packages) != 0:

            (out_dir / "git").mkdir()

            # run download jobs in parallel

            git_args_gen = ((url, git_sha_rev, out_dir) for git_sha_rev, url in git_sha_rev_to_url.items())

            pool.starmap(download_git_tree, git_args_gen)

        if len(registry_packages) != 0:

            (out_dir / "tarballs").mkdir()

            # run download jobs in parallel

            tarball_args_gen = ((pkg, out_dir) for pkg in registry_packages)

            pool.starmap(download_tarball, tarball_args_gen)

def get_manifest_metadata(manifest_path: Path) -> dict[str, Any]:

    cmd = ["cargo", "metadata", "--format-version", "1", "--no-deps", "--manifest-path", str(manifest_path)]

    output = subprocess.check_output(cmd)

    return json.loads(output)

def try_get_crate_manifest_path_from_mainfest_path(manifest_path: Path, crate_name: str) -> Path | None:

    metadata = get_manifest_metadata(manifest_path)

    for pkg in metadata["packages"]:

        if pkg["name"] == crate_name:

            return Path(pkg["manifest_path"])

    return None

def find_crate_manifest_in_tree(tree: Path, crate_name: str) -> Path:

    # in some cases Cargo.toml is not located at the top level, so we also look at subdirectories

    manifest_paths = tree.glob("**/Cargo.toml")

    for manifest_path in manifest_paths:

        res = try_get_crate_manifest_path_from_mainfest_path(manifest_path, crate_name)

        if res is not None:

            return res

    raise Exception(f"Couldn't find manifest for crate {crate_name} inside {tree}.")

def copy_and_patch_git_crate_subtree(git_tree: Path, crate_name: str, crate_out_dir: Path) -> None:

    crate_manifest_path = find_crate_manifest_in_tree(git_tree, crate_name)

    crate_tree = crate_manifest_path.parent

    eprint(f"Copying to {crate_out_dir}")

    shutil.copytree(crate_tree, crate_out_dir)

    crate_out_dir.chmod(0o755)

    with open(crate_manifest_path, "r") as f:

        manifest_data = f.read()

    if "workspace" in manifest_data:

        crate_manifest_metadata = get_manifest_metadata(crate_manifest_path)

        workspace_root = Path(crate_manifest_metadata["workspace_root"])

        root_manifest_path = workspace_root / "Cargo.toml"

        manifest_path = crate_out_dir / "Cargo.toml"

        manifest_path.chmod(0o644)

        eprint(f"Patching {manifest_path}")

        cmd = ["replace-workspace-values", str(manifest_path), str(root_manifest_path)]

        subprocess.check_output(cmd)

def extract_crate_tarball_contents(tarball_path: Path, crate_out_dir: Path) -> None:

    eprint(f"Unpacking to {crate_out_dir}")

    crate_out_dir.mkdir()

    cmd = ["tar", "xf", str(tarball_path), "-C", str(crate_out_dir), "--strip-components=1"]

    subprocess.check_output(cmd)

def create_vendor(vendor_staging_dir: Path, out_dir: Path) -> None:

    lockfile_path = vendor_staging_dir / "Cargo.lock"

    out_dir.mkdir(exist_ok=True)

    shutil.copy(lockfile_path, out_dir / "Cargo.lock")

    cargo_toml = load_toml(lockfile_path)

    config_lines = [

        '[source.vendored-sources]',

        'directory = "@vendor@"',

        '[source.crates-io]',

        'replace-with = "vendored-sources"',

    ]

    seen_source_keys = set()

    for pkg in cargo_toml["package"]:

        # ignore local dependenices

        if "source" not in pkg.keys():

            continue

        source: str = pkg["source"]

        dir_name = f"{pkg["name"]}-{pkg["version"]}"

        crate_out_dir = out_dir / dir_name

        if source.startswith("git+"):

            source_info = parse_git_source(pkg["source"])

            git_sha_rev = source_info["git_sha_rev"]

            git_tree = vendor_staging_dir / "git" / git_sha_rev

            copy_and_patch_git_crate_subtree(git_tree, pkg["name"], crate_out_dir)

            # git based crates allow having no checksum information

            with open(crate_out_dir / ".cargo-checksum.json", "w") as f:

                json.dump({"files": {}}, f)

            source_key = source[0:source.find("#")]

            if source_key in seen_source_keys:

                continue

            seen_source_keys.add(source_key)

            config_lines.append(f'[source."{source_key}"]')

            config_lines.append(f'git = "{source_info["url"]}"')

            if source_info["type"] is not None:

                config_lines.append(f'{source_info["type"]} = "{source_info["value"]}"')

            config_lines.append('replace-with = "vendored-sources"')

        elif source.startswith("registry+"):

            filename = f"{pkg["name"]}-{pkg["version"]}.tar.gz"

            tarball_path = vendor_staging_dir / "tarballs" / filename

            extract_crate_tarball_contents(tarball_path, crate_out_dir)

            # non-git based crates need the package checksum at minimum

            with open(crate_out_dir / ".cargo-checksum.json", "w") as f:

                json.dump({"files": {}, "package": pkg["checksum"]}, f)

        else:

            raise Exception(f"Can't process source: {source}.")

    (out_dir / ".cargo").mkdir()

    with open(out_dir / ".cargo" / "config.toml", "w") as config_file:

        config_file.writelines(line + "\n" for line in config_lines)

def main() -> None:

    subcommand = sys.argv[1]

    subcommand_func_dict = {

        "create-vendor-staging": lambda: create_vendor_staging(lockfile_path=Path(sys.argv[2]), out_dir=Path(sys.argv[3])),

        "create-vendor": lambda: create_vendor(vendor_staging_dir=Path(sys.argv[2]), out_dir=Path(sys.argv[3]))

    }

    subcommand_func = subcommand_func_dict.get(subcommand)

    if subcommand_func is None:

        raise Exception(f"Unknown subcommand: '{subcommand}'. Must be one of {list(subcommand_func_dict.keys())}")

    subcommand_func()

if __name__ == "__main__":

    main()

{

  lib,

  stdenvNoCC,

  runCommand,

  writers,

  python3Packages,

  cargo,

  nix-prefetch-git,

  cacert,

}:

let

  replaceWorkspaceValues = writers.writePython3Bin "replace-workspace-values" {

    libraries = with python3Packages; [

      tomli

      tomli-w

    ];

    flakeIgnore = [

      "E501"

      "W503"

    ];

  } (builtins.readFile ./replace-workspace-values.py);

  fetchCargoVendorUtil = writers.writePython3Bin "fetch-cargo-vendor-util" {

    libraries = with python3Packages; [

      requests

    ];

    flakeIgnore = [

      "E501"

    ];

  } (builtins.readFile ./fetch-cargo-vendor-util.py);

in

{

  name ? if args ? pname && args ? version then "${args.pname}-${args.version}" else "cargo-deps",

  hash ? (throw "fetchCargoVendor requires a `hash` value to be set for ${name}"),

  nativeBuildInputs ? [ ],

  ...

}@args:

# TODO: add asserts about pname version and name

let

  removedArgs = [

    "name"

    "pname"

    "version"

    "nativeBuildInputs"

    "hash"

  ];

  vendorStaging = stdenvNoCC.mkDerivation (

    {

      name = "${name}-vendor-staging";

      nativeBuildInputs = [

        fetchCargoVendorUtil

        nix-prefetch-git

        cacert

      ] ++ nativeBuildInputs;

      buildPhase = ''

        runHook preBuild

        fetch-cargo-vendor-util create-vendor-staging ./Cargo.lock "$out"

        runHook postBuild

      '';

      dontInstall = true;

      dontFixup = true;

      outputHash = hash;

      outputHashAlgo = if hash == "" then "sha256" else null;

      outputHashMode = "recursive";

    }

    // builtins.removeAttrs args removedArgs

  );

in

runCommand "${name}-vendor"

  {

    inherit vendorStaging;

    nativeBuildInputs = [

      fetchCargoVendorUtil

      cargo

      replaceWorkspaceValues

    ];

  }

  ''

    fetch-cargo-vendor-util create-vendor "$vendorStaging" "$out"

  ''