Unverified Commit 993c8f16 authored by Zeke Dou's avatar Zeke Dou Committed by GitHub
Browse files

mixRelease: improve the implementation (#266397) 



* mixRelease: format code by nixpkgs-format

* mixRelease: investigate why erlang is referenced in resulting derivation

* mixRelease: organize nativeBuildInputs and buildInputs

It:
+ organizes `nativeBuildInputs` in a structured way.
+ moves `builtins.attrValues mixNixDeps` to `nativeBuildInputs`, because it's only used in compile-time

* mixRelease: remove current attempt for removing erlang references in resulting derivation

As said in the comment about "remove erlang references in resulting
derivation", for now, we don't have a robust method to do that.

Although these removed code did some work, they did not achieve the
final goal - remove erlang references in resulting derivation.
Therefore, it is better to remove them and provide future implementation
with a clean foundation.

> If you want to find these old codes, you can also retrieve them from
> the git history.

* mixRelease: remove all files for Microsoft Windows

* mixRelease: add new option - `removeCookie`

* mixRelease: polish comments

+ Capitalize the sentences.
+ Add punctuation marks.
+ Format a little code.

* mixRelease: wrap programs in $out/bin with their runtime deps

* mixRelease: commit what happysalada suggests

---------

Co-authored-by: default avatarc4710n <c4710n@users.noreply.github.com>
parent 201f93fb

pkgs/development/beam-modules/mix-release.nix

+104 −58
Original line number Diff line number Diff line 
{ stdenv, lib, elixir, erlang, findutils, hex, rebar, rebar3, fetchMixDeps, makeWrapper, git, ripgrep }@inputs:

{ stdenv

, lib

, elixir

, erlang

, hex

, git

, rebar

, rebar3

, fetchMixDeps

, findutils

, makeWrapper

, coreutils

, gnused

, gnugrep

, gawk

}@inputs:



{ pname

, version
@@ -10,18 +25,34 @@ 
, mixEnv ? "prod"

, compileFlags ? [ ]



  # mix fixed output derivation dependencies

  # Mix dependencies provided as a fixed output derivation

, mixFodDeps ? null



  # mix dependencies generated by mix2nix

  # this assumes each dependency is built by buildMix or buildRebar3

  # each dependency needs to have a setup hook to add the lib path to $ERL_LIBS

  # this is how mix will find dependencies

  # Mix dependencies generated by mix2nix

  #

  # This assumes each dependency is built by buildMix or buildRebar3. Each

  # dependency needs to have a setup hook to add the lib path to $ERL_LIBS.

  # This is how Mix finds dependencies.

, mixNixDeps ? { }



, elixir ? inputs.elixir

, hex ? inputs.hex.override { inherit elixir; }



  # Remove releases/COOKIE

  #

  # People have different views on the nature of cookies. Some believe that they are

  # secrets, while others believe they are just ids for clustering nodes instead of

  # secrets.

  #

  # If you think cookie is secret, you can set this attr to true, then it will be

  # removed from nix store. If not, you can set it to false.

  #

  # For backward compatibility, it is set to true by default.

  #

  # You can always specify a custom cookie by using RELEASE_COOKIE environment

  # variable, regardless of the value of this attr.

, removeCookie ? true



  # This reduces closure size, but can lead to some hard to understand runtime

  # errors, so use with caution. See e.g.

  # https://github.com/whitfin/cachex/issues/205
@@ -31,59 +62,67 @@ 
, ...

}@attrs:

let

  # remove non standard attributes that cannot be coerced to strings

  # Remove non standard attributes that cannot be coerced to strings

  overridable = builtins.removeAttrs attrs [ "compileFlags" "mixNixDeps" ];

in

assert mixNixDeps != { } -> mixFodDeps == null;

assert stripDebug -> !enableDebugInfo;



stdenv.mkDerivation (overridable // {

  # rg is used as a better grep to search for erlang references in the final release

  nativeBuildInputs = nativeBuildInputs ++ [ erlang hex elixir makeWrapper git ripgrep ];

  buildInputs = buildInputs ++ builtins.attrValues mixNixDeps;

  nativeBuildInputs = nativeBuildInputs ++

    # Erlang/Elixir deps

    [ erlang elixir hex git ] ++

    # Mix deps

    (builtins.attrValues mixNixDeps) ++

    # other compile-time deps

    [ findutils makeWrapper ];



  buildInputs = buildInputs;



  MIX_ENV = mixEnv;

  MIX_DEBUG = if enableDebugInfo then 1 else 0;

  HEX_OFFLINE = 1;



  DEBUG = if enableDebugInfo then 1 else 0; # for Rebar3 compilation

  # the api with `mix local.rebar rebar path` makes a copy of the binary

  # some older dependencies still use rebar

  # The API with `mix local.rebar rebar path` makes a copy of the binary

  # some older dependencies still use rebar.

  MIX_REBAR = "${rebar}/bin/rebar";

  MIX_REBAR3 = "${rebar3}/bin/rebar3";



  LC_ALL = "C.UTF-8";



  postUnpack = ''

    export HEX_HOME="$TEMPDIR/hex"

    # Mix and Hex

    export MIX_HOME="$TEMPDIR/mix"

    export HEX_HOME="$TEMPDIR/hex"



    # Rebar

    export REBAR_GLOBAL_CONFIG_DIR="$TEMPDIR/rebar3"

    export REBAR_CACHE_DIR="$TEMPDIR/rebar3.cache"



    ${lib.optionalString (mixFodDeps != null) ''

      # compilation of the dependencies will require

      # that the dependency path is writable

      # thus a copy to the TEMPDIR is inevitable here

      # Compilation of the dependencies will require that the dependency path is

      # writable, thus a copy to the $TEMPDIR is inevitable here.

      export MIX_DEPS_PATH="$TEMPDIR/deps"

      cp --no-preserve=mode -R "${mixFodDeps}" "$MIX_DEPS_PATH"

    ''

    }



    ''}

  '' + (attrs.postUnpack or "");



  configurePhase = attrs.configurePhase or ''

    runHook preConfigure



    ${./mix-configure-hook.sh}

    # this is needed for projects that have a specific compile step



    # This is needed for projects that have a specific compile step

    # the dependency needs to be compiled in order for the task

    # to be available

    # Phoenix projects for example will need compile.phoenix

    # to be available.

    #

    # Phoenix projects for example will need compile.phoenix.

    mix deps.compile --no-deps-check --skip-umbrella-children



    # Symlink dependency sources. This is needed for projects that require

    # access to the source of their dependencies. For example, Phoenix

    # applications need javascript assets to build asset bundles.

    # projects need javascript assets to build asset bundles.

    ${lib.optionalString (mixNixDeps != { }) ''

      mkdir -p deps
@@ -113,7 +152,6 @@ stdenv.mkDerivation (overridable // { 
    runHook postBuild

  '';





  installPhase = attrs.installPhase or ''

    runHook preInstall
@@ -122,42 +160,50 @@ stdenv.mkDerivation (overridable // { 
    runHook postInstall

  '';



  # Stripping of the binary is intentional

  # even though it does not affect beam files

  # it is necessary for NIFs binaries

  postFixup = ''

    if [ -e "$out/bin/${pname}.bat" ]; then # absent in special cases, i.e. elixir-ls

      rm "$out/bin/${pname}.bat" # windows file

    fi

    # contains secrets and should not be in the nix store

    # TODO document how to handle RELEASE_COOKIE

    # secrets should not be in the nix store.

    # This is only used for connecting multiple nodes

    if [ -e $out/releases/COOKIE ]; then # absent in special cases, i.e. elixir-ls

      rm $out/releases/COOKIE

    fi

    # removing unused erlang reference from resulting derivation to reduce

    # closure size

    if [ -e $out/erts-* ]; then

      echo "ERTS found in $out - removing references to erlang to reduce closure size"

      # there is a link in $out/erts-*/bin/start always

      # TODO:

      # sometimes there are links in dependencies like bcrypt compiled binaries

      # at the moment those are not removed since substituteInPlace will

      # error on binaries

      for file in $(rg "${erlang}/lib/erlang" "$out" --files-with-matches); do

        echo "removing reference to erlang in $file"

        substituteInPlace "$file" --replace "${erlang}/lib/erlang" "$out"

    # Remove files for Microsoft Windows

    rm -f "$out"/bin/*.bat



    # Wrap programs in $out/bin with their runtime deps

    for f in $(find $out/bin/ -type f -executable); do

      wrapProgram "$f" \

        --prefix PATH : ${lib.makeBinPath [

          coreutils

          gnused

          gnugrep

          gawk

        ]}

    done

  '' + lib.optionalString removeCookie ''

    if [ -e $out/releases/COOKIE ]; then

      rm $out/releases/COOKIE

    fi

  '' + lib.optionalString stripDebug ''

    # strip debug symbols to avoid hardreferences to "foreign" closures actually

    # Strip debug symbols to avoid hardreferences to "foreign" closures actually

    # not needed at runtime, while at the same time reduce size of BEAM files.

    erl -noinput -eval 'lists:foreach(fun(F) -> io:format("Stripping ~p.~n", [F]), beam_lib:strip(F) end, filelib:wildcard("'"$out"'/**/*.beam"))' -s init stop

  '';



  # TODO investigate why the resulting closure still has

  # a reference to erlang.

  # uncommenting the following will fail the build

  # TODO: remove erlang references in resulting derivation

  #

  # # Step 1 - investigate why the resulting derivation still has references to erlang.

  #

  # The reason is that the generated binaries contains erlang reference. Here's a repo to

  # demonstrate the problem - <https://github.com/plastic-gun/nix-mix-release-unwanted-references>.

  #

  #

  # # Step 2 - remove erlang references from the binaries

  #

  # As said in above repo, it's hard to remove erlang references from `.beam` binaries.

  #

  # We need more experienced developers to resolve this issue.

  #

  #

  # # Tips

  #

  # When resolving this issue, it is convenient to fail the build when erlang is referenced,

  # which can be achieved by using:

  #

  #   disallowedReferences = [ erlang ];

  #

})