Unverified Commit b9a22852 authored by Nick Cao's avatar Nick Cao Committed by GitHub
Browse files

Julia.withPackages improvements (#433332)

parents 5e6b1595 338b5bde

pkgs/development/julia-modules/default.nix

+33 −9
Original line number Diff line number Diff line
@@ -11,6 +11,7 @@ 


  # Artifacts dependencies

  fetchurl,

  gcc,

  glibc,

  pkgs,

  stdenv,
@@ -79,7 +80,9 @@ let 
    PythonCall = [ "PyCall" ];

  };



  # Invoke Julia resolution logic to determine the full dependency closure

  # Invoke Julia resolution logic to determine the full dependency closure. Also

  # gather information on the Julia standard libraries, which we'll need to

  # generate a Manifest.toml.

  packageOverridesRepoified = lib.mapAttrs util.repoifySimple packageOverrides;

  closureYaml = callPackage ./package-closure.nix {

    inherit
@@ -90,6 +93,9 @@ let 
      ;

    packageOverrides = packageOverridesRepoified;

  };

  stdlibInfos = callPackage ./stdlib-infos.nix {

    inherit julia;

  };



  # Generate a Nix file consisting of a map from dependency UUID --> package info with fetchgit call:

  # {
@@ -181,6 +187,27 @@ let 
          "${dependencyUuidToRepoYaml}" \

          "$out"

      '';

  project =

    runCommand "julia-project"

      {

        buildInputs = [

          (python3.withPackages (

            ps: with ps; [

              toml

              pyyaml

            ]

          ))

          git

        ];

      }

      ''

        python ${./python}/project.py \

          "${closureYaml}" \

          "${stdlibInfos}" \

          '${lib.generators.toJSON { } overridesOnly}' \

          "${dependencyUuidToRepoYaml}" \

          "$out"

      '';



  # Next, deal with artifacts. Scan each artifacts file individually and generate a Nix file that

  # produces the desired Overrides.toml.
@@ -220,7 +247,7 @@ let 
        ;

    }

    // lib.optionalAttrs (!stdenv.targetPlatform.isDarwin) {

      inherit glibc;

      inherit gcc glibc;

    }

  );

  overridesJson = writeTextFile {
@@ -235,8 +262,7 @@ let 
          "$out"

      '';



  # Build a Julia project and depot. The project contains Project.toml/Manifest.toml, while the

  # depot contains package build products (including the precompiled libraries, if precompile=true)

  # Build a Julia project and depot under $out/project and $out/depot respectively

  projectAndDepot = callPackage ./depot.nix {

    inherit

      closureYaml
@@ -247,12 +273,8 @@ let 
      precompile

      ;

    julia = juliaWrapped;

    inherit project;

    registry = minimalRegistry;

    packageNames =

      if makeTransitiveDependenciesImportable then

        lib.mapAttrsToList (uuid: info: info.name) dependencyUuidToInfo

      else

        packageNames;

  };



in
@@ -276,7 +298,9 @@ runCommand "julia-${julia.version}-env" 
      inherit artifactsNix;

      inherit overridesJson;

      inherit overridesToml;

      inherit project;

      inherit projectAndDepot;

      inherit stdlibInfos;

    };

  }

  (

pkgs/development/julia-modules/depot.nix

+18 −21
Original line number Diff line number Diff line
@@ -14,7 +14,7 @@ 
  juliaCpuTarget,

  overridesToml,

  packageImplications,

  packageNames,

  project,

  precompile,

  registry,

}:
@@ -44,7 +44,7 @@ runCommand "julia-depot" 
      (python3.withPackages (ps: with ps; [ pyyaml ]))

    ]

    ++ extraLibs;

    inherit precompile registry;

    inherit precompile project registry;

  }

  (

    ''
@@ -52,19 +52,21 @@ runCommand "julia-depot" 


      echo "Building Julia depot and project with the following inputs"

      echo "Julia: ${julia}"

      echo "Project: $project"

      echo "Registry: $registry"

      echo "Overrides ${overridesToml}"



      mkdir -p $out/project

      export JULIA_PROJECT="$out/project"

      cp "$project/Manifest.toml" "$JULIA_PROJECT/Manifest.toml"

      cp "$project/Project.toml" "$JULIA_PROJECT/Project.toml"



      mkdir -p $out/depot/artifacts

      export JULIA_DEPOT_PATH="$out/depot"

      cp ${overridesToml} $out/depot/artifacts/Overrides.toml



      # These can be useful to debug problems

      # export JULIA_DEBUG=Pkg

      # export JULIA_DEBUG=loading

      # export JULIA_DEBUG=Pkg,loading



      ${setJuliaSslCaRootsPath}
@@ -104,19 +106,15 @@ runCommand "julia-depot" 


        Pkg.Registry.add(Pkg.RegistrySpec(path="${registry}"))



        input = ${lib.generators.toJSON { } packageNames} ::Vector{String}



        if isfile("extra_package_names.txt")

          append!(input, readlines("extra_package_names.txt"))

        end



        input = unique(input)



        if !isempty(input)

          println("Adding packages: " * join(input, " "))

          Pkg.add(input; preserve=PRESERVE_NONE)

        # No need to Pkg.activate() since we set JULIA_PROJECT above

        println("Running Pkg.instantiate()")

        Pkg.instantiate()



        # Build is a separate step from instantiate.

        # Needed for packages like Conda.jl to set themselves up.

        println("Running Pkg.build()")

        Pkg.build()



        if "precompile" in keys(ENV) && ENV["precompile"] != "0" && ENV["precompile"] != ""

          if isdefined(Sys, :CPU_NAME)

            println("Precompiling with CPU_NAME = " * Sys.CPU_NAME)
@@ -124,7 +122,6 @@ runCommand "julia-depot" 


          Pkg.precompile()

        end

        end



        # Remove the registry to save space

        Pkg.Registry.rm("General")

pkgs/development/julia-modules/package-closure.nix

+9 −0
Original line number Diff line number Diff line
@@ -43,12 +43,21 @@ let 
            println(io, "- name: " * spec.name)

            println(io, "  uuid: " * string(spec.uuid))

            println(io, "  version: " * string(spec.version))

            println(io, "  tree_hash: " * string(spec.tree_hash))

            if endswith(spec.name, "_jll") && haskey(deps_map, spec.uuid)

                println(io, "  depends_on: ")

                for (dep_name, dep_uuid) in pairs(deps_map[spec.uuid])

                    println(io, "    \"$(dep_name)\": \"$(dep_uuid)\"")

                end

            end

            println(io, "  deps: ")

            for (dep_name, dep_uuid) in pairs(deps_map[spec.uuid])

                println(io, "  - name: \"$(dep_name)\"")

                println(io, "    uuid: \"$(dep_uuid)\"")

            end

            if spec.name in input

                println(io, "  is_input: true")

            end

        end

    end

  '';

pkgs/development/julia-modules/python/extract_artifacts.py

+6 −3
Original line number Diff line number Diff line
@@ -47,14 +47,17 @@ def get_archive_derivation(uuid, artifact_name, url, sha256, closure_dependencie 


            ''"""

  else:

    # We provide gcc.cc.lib by default in order to get some common libraries

    # like libquadmath.so. A number of packages expect this to be available and

    # will give linker errors if it isn't.

    fixup = f"""fixupPhase = let

            libs = lib.concatMap (lib.mapAttrsToList (k: v: v.path))

                               [{" ".join(["uuid-" + x for x in depends_on])}];

            in ''

              find $out -type f -executable -exec \

                patchelf --set-rpath \$ORIGIN:\$ORIGIN/../lib:${{lib.makeLibraryPath (["$out" glibc] ++ libs ++ (with pkgs; [{" ".join(other_libs)}]))}} {{}} \;

                patchelf --set-rpath \\$ORIGIN:\\$ORIGIN/../lib:${{lib.makeLibraryPath (["$out" glibc gcc.cc.lib] ++ libs ++ (with pkgs; [{" ".join(other_libs)}]))}} {{}} \\;

              find $out -type f -executable -exec \

                patchelf --set-interpreter ${{glibc}}/lib/ld-linux-x86-64.so.2 {{}} \;

                patchelf --set-interpreter ${{glibc}}/lib/ld-linux-x86-64.so.2 {{}} \\;

            ''"""



  return f"""stdenv.mkDerivation {{
@@ -145,7 +148,7 @@ def main(): 
    if is_darwin:

      f.write("{ lib, fetchurl, pkgs, stdenv }:\n\n")

    else:

      f.write("{ lib, fetchurl, glibc, pkgs, stdenv }:\n\n")

      f.write("{ lib, fetchurl, gcc, glibc, pkgs, stdenv }:\n\n")



    f.write("rec {\n")

pkgs/development/julia-modules/python/minimal_registry.py

+40 −11
Original line number Diff line number Diff line
@@ -31,7 +31,8 @@ with open(dependencies_path, "r") as f: 


os.makedirs(out_path)



registry = toml.load(registry_path / "Registry.toml")

full_registry = toml.load(registry_path / "Registry.toml")

registry = full_registry.copy()

registry["packages"] = {k: v for k, v in registry["packages"].items() if k in uuid_to_versions}



for (uuid, versions) in uuid_to_versions.items():
@@ -80,20 +81,48 @@ for (uuid, versions) in uuid_to_versions.items(): 
      if (registry_path / path / f).exists():

        shutil.copy2(registry_path / path / f, out_path / path)



    # Copy the Versions.toml file, trimming down to the versions we care about

    # Copy the Versions.toml file, trimming down to the versions we care about.

    # In the case where versions=None, this is a weak dep, and we keep all versions.

    all_versions = toml.load(registry_path / path / "Versions.toml")

    versions_to_keep = {k: v for k, v in all_versions.items() if k in versions}

    versions_to_keep = {k: v for k, v in all_versions.items() if k in versions} if versions != None else all_versions

    for k, v in versions_to_keep.items():

      del v["nix-sha256"]

    with open(out_path / path / "Versions.toml", "w") as f:

      toml.dump(versions_to_keep, f)



    if versions is None:

      # This is a weak dep; just grab the whole Package.toml

      shutil.copy2(registry_path / path / "Package.toml", out_path / path / "Package.toml")

    elif uuid in uuid_to_store_path:

      # Fill in the local store path for the repo

    if not uuid in uuid_to_store_path: continue

      package_toml = toml.load(registry_path / path / "Package.toml")

      package_toml["repo"] = "file://" + uuid_to_store_path[uuid]

      with open(out_path / path / "Package.toml", "w") as f:

        toml.dump(package_toml, f)



# Look for missing weak deps and include them. This can happen when our initial

# resolve step finds dependencies, but we fail to resolve them at the project.py

# stage. Usually this happens because the package that depends on them does so

# as a weak dep, but doesn't have a Package.toml in its repo making this clear.

for pkg in desired_packages:

  for dep in (pkg.get("deps", []) or []):

    uuid = dep["uuid"]

    if not uuid in uuid_to_versions:

      entry = full_registry["packages"].get(uuid)

      if not entry:

        print(f"""WARNING: found missing UUID but couldn't resolve it: {uuid}""")

        continue



      # Add this entry back to the minimal Registry.toml

      registry["packages"][uuid] = entry



      # Bring over the Package.toml

      path = Path(entry["path"])

      if (out_path / path / "Package.toml").exists():

        continue

      Path(out_path / path).mkdir(parents=True, exist_ok=True)

      shutil.copy2(registry_path / path / "Package.toml", out_path / path / "Package.toml")



# Finally, dump the Registry.toml

with open(out_path / "Registry.toml", "w") as f:

    toml.dump(registry, f)