Commit bbd0655a authored by Motiejus Jakštys's avatar Motiejus Jakštys
Browse files

add compressDrv and compressDrvWeb

*compressDrv* compresses files in a given derivation.

*compressDrvWeb* compresses a derivation for a loosely-defined
pre-compressed "web server" usage.

This intends to replace the `passthru.data-compressed` derivations that
have accumulated in nixpkgs with something more reusable.
parent e348e455
Loading
Loading
Loading
Loading
+2 −0
Original line number Diff line number Diff line
@@ -13,6 +13,8 @@
- `authelia` has been upgraded to version 4.38. This version brings several features and improvements which are detailed in the [release blog post](https://www.authelia.com/blog/4.38-release-notes/).
  This release also deprecates some configuration keys, which are likely to be removed in future version 5.0, but they are still supported and expected to be working in the current version.

- `compressDrv` can compress selected files in a derivation. `compressDrvWeb` compresses files for common web server usage (`.gz` with `zopfli`, `.br` with `brotli`).

- `hardware.display` is a new module implementing workarounds for misbehaving monitors
  through setting up custom EDID files and forcing kernel/framebuffer modes.

+63 −0
Original line number Diff line number Diff line
{
  lib,
  xorg,
  runCommand,
}:
/**
  #  compressDrv compresses files in a given derivation.

  ## Inputs:

  `formats` ([String])

  : List of file extensions to compress. Example: `["txt" "svg" "xml"]`.

  `compressors` (String -> String)

  : Map a desired extension (e.g. `gz`) to a compress program.

  The compressor program that will be executed to get the `COMPRESSOR` extension.
  The program should have a single " {}", which will be the replaced with the
  target filename.

  Compressor must:
  - read symlinks (thus --force is needed to gzip, zstd, xz).
  - keep the original file in place (--keep).

  Example:

  ```
  {
    xz = "${xz}/bin/xz --force --keep {}";
  }
  ```

  See compressDrvWeb, which is a wrapper on top of compressDrv, for broader use
  examples.
*/
drv:
{ formats, compressors, ... }:
let
  validProg =
    ext: prog:
    let
      matches = (builtins.length (builtins.split "\\{}" prog) - 1) / 2;
    in
    lib.assertMsg (
      matches == 1
    ) "compressor ${ext} needs to have exactly one '{}', found ${builtins.toString matches}";
  mkCmd =
    ext: prog:
    assert validProg ext prog;
    ''
      find -L $out -type f -regextype posix-extended -iregex '.*\.(${formatsPipe})' -print0 \
        | xargs -0 -P$NIX_BUILD_CORES -I{} ${prog}
    '';
  formatsPipe = builtins.concatStringsSep "|" formats;
in
runCommand "${drv.name}-compressed" { } ''
  mkdir $out
  (cd $out; ${xorg.lndir}/bin/lndir ${drv})

  ${lib.concatStringsSep "\n\n" (lib.mapAttrsToList mkCmd compressors)}
''
+42 −0
Original line number Diff line number Diff line
{
  gzip,
  runCommand,
  compressDrv,
}:
let
  example = runCommand "sample-drv" { } ''
    mkdir $out
    echo 42 > $out/1.txt
    echo 43 > $out/1.md
    touch $out/2.png
  '';
  drv = compressDrv example {
    formats = [ "txt" ];
    compressors.gz = "${gzip}/bin/gzip --force --keep --fast {}";
  };
  wrapped = compressDrv drv {
    formats = [ "md" ];
    compressors.gz = "${gzip}/bin/gzip --force --keep --fast {}";
  };
in
runCommand "test-compressDrv" { } ''
  set -ex

  ls -l ${drv}
  test -h ${drv}/1.txt
  test -f ${drv}/1.txt.gz
  cmp ${drv}/1.txt <(${gzip}/bin/zcat ${drv}/1.txt.gz)

  test -h ${drv}/2.png
  test ! -a ${drv}/2.png.gz

  # compressDrv always points to the final file, no matter how many times
  # it's been wrapped
  cmp <(readlink -e ${drv}/1.txt) <(readlink -e ${wrapped}/1.txt)

  test -f ${wrapped}/1.txt.gz
  test -f ${wrapped}/1.md.gz
  test ! -f ${drv}/1.md.gz

  mkdir $out
''
+103 −0
Original line number Diff line number Diff line
{
  zopfli,
  brotli,
  compressDrv,
}:
/**
  # compressDrvWeb compresses a derivation for common web server use.

  Useful when one wants to pre-compress certain static assets and pass them to
  the web server. For example, `pkgs.gamja` creates this derivation:

      /nix/store/2wn1qbk8gp4y2m8xvafxv1b2dcdqj8fz-gamja-1.0.0-beta.9/
      ├── index.2fd01148.js
      ├── index.2fd01148.js.map
      ├── index.37aa9a8a.css
      ├── index.37aa9a8a.css.map
      ├── index.html
      └── manifest.webmanifest

  `pkgs.compressDrvWeb pkgs.gamja`:

      /nix/store/f5ryid7zrw2hid7h9kil5g5j29q5r2f7-gamja-1.0.0-beta.9-compressed
      ├── index.2fd01148.js -> /nix/store/2wn1qbk8gp4y2m8xvafxv1b2dcdqj8fz-gamja-1.0.0-beta.9/index.2fd01148.js
      ├── index.2fd01148.js.br
      ├── index.2fd01148.js.gz
      ├── index.2fd01148.js.map -> /nix/store/2wn1qbk8gp4y2m8xvafxv1b2dcdqj8fz-gamja-1.0.0-beta.9/index.2fd01148.js.map
      ├── index.2fd01148.js.map.br
      ├── index.2fd01148.js.map.gz
      ├── index.37aa9a8a.css -> /nix/store/2wn1qbk8gp4y2m8xvafxv1b2dcdqj8fz-gamja-1.0.0-beta.9/index.37aa9a8a.css
      ├── index.37aa9a8a.css.br
      ├── index.37aa9a8a.css.gz
      ├── index.37aa9a8a.css.map -> /nix/store/2wn1qbk8gp4y2m8xvafxv1b2dcdqj8fz-gamja-1.0.0-beta.9/index.37aa9a8a.css.map
      ├── index.37aa9a8a.css.map.br
      ├── index.37aa9a8a.css.map.gz
      ├── index.html -> /nix/store/2wn1qbk8gp4y2m8xvafxv1b2dcdqj8fz-gamja-1.0.0-beta.9/index.html
      ├── index.html.br
      ├── index.html.gz
      ├── manifest.webmanifest -> /nix/store/2wn1qbk8gp4y2m8xvafxv1b2dcdqj8fz-gamja-1.0.0-beta.9/manifest.webmanifest
      ├── manifest.webmanifest.br
      └── manifest.webmanifest.gz

  When this `-compressed` directory is passed to a properly configured web
  server, it will serve those pre-compressed files:

      $ curl -I -H 'Accept-Encoding: br' https://irc.example.org/
      <...>
      content-encoding: br
      <...>

  For example, a caddy configuration snippet for gamja to serve
  the static assets (JS, CSS files) pre-compressed:

      virtualHosts."irc.example.org".extraConfig = ''
        root * ${pkgs.compressDrvWeb pkgs.gamja {}}
        file_server browse {
            precompressed br gzip
        }
      '';

  This feature is also available in nginx via `ngx_brotli` and
  `ngx_http_gzip_static_module`.

  ## Inputs

  `formats` ([String])

  : List of file extensions to compress. Default is common formats that compress
  well. The list may be expanded.

  `extraFormats` ([String])

  : Extra extensions to compress in addition to `formats`.

  `compressors` (String -> String)

  : See parameter `compressors` of compressDrv.
*/
drv:
{
  formats ? [
    "css"
    "js"
    "svg"
    "ttf"
    "eot"
    "txt"
    "xml"
    "map"
    "html"
    "json"
    "webmanifest"
  ],
  extraFormats ? [ ],
  compressors ? {
    "gz" = "${zopfli}/bin/zopfli --keep {}";
    "br" = "${brotli}/bin/brotli --keep --no-copy-stat {}";
  },
  ...
}:
compressDrv drv {
  formats = formats ++ extraFormats;
  compressors = compressors;
}
+2 −0
Original line number Diff line number Diff line
@@ -106,6 +106,8 @@ with pkgs;
  cc-multilib-gcc = callPackage ./cc-wrapper/multilib.nix { stdenv = gccMultiStdenv; };
  cc-multilib-clang = callPackage ./cc-wrapper/multilib.nix { stdenv = clangMultiStdenv; };

  compress-drv = callPackage ../build-support/compress-drv/test.nix { };

  fetchurl = callPackages ../build-support/fetchurl/tests.nix { };
  fetchtorrent = callPackages ../build-support/fetchtorrent/tests.nix { };
  fetchpatch = callPackages ../build-support/fetchpatch/tests.nix { };
Loading