Unverified Commit e1578315 authored by emilylange's avatar emilylange
Browse files

chromium: improve and move `recompressTarball` 

Recap: We need that (arguably stupid) helper function/drv because the
chromium tarball is big -- and is likely to increase even more in the
future. So big, that we eventually exceeded hydra.nixos.org's
max-output-limit (3G).  Instead of raising global hydra's limit, it was
decided that we recompress the tarball after deleting unused vendored
files from it.

I spent a lot of time on a version/prototype that does everything
(downloading, decompression, tar extraction, deleting unused files,
reproducible tar recreation and finally recompression) via stdin but
eventually had to scratch that.

GNU tar does not allow to create a tarball just from stdin, nixpkgs'
stdenv isn't built with stdin/stdout/pipes in mind, and things a lot of
other things I probably already forgot.

Nonetheless, this version improves multiple things:
- No more `mv` (used to be multiple, not just ours, since fetchzip had
  some as well)
- No more `rm` to get rid of the extracted files before recompressing.
  Instead, we simply don't extract them in the first place (thanks to
  tar's --exlude).
- No more "no space left" that happened due to `downloadToTemp = true;`.
- Multithreaded xz decompression, since that commit is still in
  staging-next.

We cannot use stdenv's unpackFile() because that does not allow us to
specify the needed --exclude (and --strip-components=1 if we don't want
to rely on glob matching).

The hash changed because we now have a static base directory ("source")
in the tarball, instead of whatever upstream provided us with (e.g.
"chromium-120.0.6099.129").
parent 0b15d0fb

pkgs/applications/networking/browsers/chromium/README.md

+0 −10
Original line number Diff line number Diff line
@@ -39,16 +39,6 @@ update `upstream-info.nix`. After updates it is important to test at least 
`nixosTests.chromium` (or basic manual testing) and `google-chrome` (which

reuses `upstream-info.nix`).



Note: Due to the script downloading many large tarballs it might be

necessary to adjust the available tmpfs size (it defaults to 10% of the

systems memory)



```nix

services.logind.extraConfig = ''

  RuntimeDirectorySize=4G

'';

```



Note: The source tarball is often only available a few hours after the release

was announced. The CI/CD status can be tracked here:

- https://ci.chromium.org/p/infra/builders/cron/publish_tarball

pkgs/applications/networking/browsers/chromium/common.nix

+1 −28
Original line number Diff line number Diff line 
{ stdenv, lib, fetchurl, fetchpatch

, fetchzip, zstd

, recompressTarball

, buildPackages

, pkgsBuildBuild

, pkgsBuildTarget
@@ -148,33 +148,6 @@ let 
      else throw "no chromium Rosetta Stone entry for os: ${platform.config}";

  };



  recompressTarball = { version, hash ? "" }: fetchzip {

    name = "chromium-${version}.tar.zstd";

    url = "https://commondatastorage.googleapis.com/chromium-browser-official/chromium-${version}.tar.xz";

    inherit hash;



    nativeBuildInputs = [ zstd ];



    postFetch = ''

      echo removing unused code from tarball to stay under hydra limit

      rm -r $out/third_party/{rust-src,llvm}



      echo moving remains out of \$out

      mv $out source



      echo recompressing final contents into new tarball

      # try to make a deterministic tarball

      tar \

        --use-compress-program "zstd -T$NIX_BUILD_CORES" \

        --sort name \

        --mtime 1970-01-01 \

        --owner=root --group=root \

        --numeric-owner --mode=go=rX,u+rw,a-s \

        -cf $out source

    '';

  };





  base = rec {

    pname = "${lib.optionalString ungoogled "ungoogled-"}${packageName}-unwrapped";

    inherit (upstream-info) version;

pkgs/applications/networking/browsers/chromium/default.nix

+1 −0
Original line number Diff line number Diff line
@@ -59,6 +59,7 @@ let 
          inherit (upstream-info.deps.gn) url rev hash;

        };

      });

      recompressTarball = callPackage ./recompress-tarball.nix { };

    });



    browser = callPackage ./browser.nix {

pkgs/applications/networking/browsers/chromium/recompress-tarball.nix

0 → 100644
+47 −0
Original line number Diff line number Diff line 
{ zstd

, fetchurl

}:



{ version

, hash ? ""

, ...

} @ args:



fetchurl ({

  name = "chromium-${version}.tar.zstd";

  url = "https://commondatastorage.googleapis.com/chromium-browser-official/chromium-${version}.tar.xz";

  inherit hash;



  # chromium xz tarballs are multiple gigabytes big and are sometimes downloaded multiples

  # times for different versions as part of our update script.

  # We originally inherited fetchzip's default for downloadToTemp (true).

  # Given the size of the /run/user tmpfs used defaults to logind's RuntimeDirectorySize=,

  # which in turn defaults to 10% of the total amount of physical RAM, this often lead to

  # "no space left" errors, eventually resulting in its own section in our chromium

  # README.md (for users wanting to run the update script).

  # Nowadays, we use fetchurl instead of fetchzip, which defaults to false instead of true.

  # We just want to be explicit and provide a place to document the history and reasoning

  # behind this.

  downloadToTemp = false;



  nativeBuildInputs = [ zstd ];



  postFetch = ''

    cat "$downloadedFile" \

    | xz -d --threads=$NIX_BUILD_CORES \

    | tar xf - \

      --warning=no-timestamp \

      --one-top-level=source \

      --exclude=third_party/llvm \

      --exclude=third_party/rust-src \

      --strip-components=1



    tar \

      --use-compress-program "zstd -T$NIX_BUILD_CORES" \

      --sort name \

      --mtime "1970-01-01" \

      --owner=root --group=root \

      --numeric-owner --mode=go=rX,u+rw,a-s \

      -cf $out source

  '';

} // removeAttrs args [ "version" ])

pkgs/applications/networking/browsers/chromium/upstream-info.nix

+2 −2
Original line number Diff line number Diff line
@@ -15,7 +15,7 @@ 
        version = "2023-10-23";

      };

    };

    hash = "sha256-+T2TOLwIwFxVDae7MFDZrjREGF+3Zx2xt/Dlu7uZggc=";

    hash = "sha256-6RURdPU1k3GaQAgA1LMQ0NhSGBEpOEJBPvk2QjLdoHo=";

    hash_deb_amd64 = "sha256-0FB1gTbsjqFRy0ocE0w5ACtD9kSJ5AMnxg+qBxqCulc=";

    version = "120.0.6099.129";

  };
@@ -32,7 +32,7 @@ 
        rev = "120.0.6099.129-1";

      };

    };

    hash = "sha256-+T2TOLwIwFxVDae7MFDZrjREGF+3Zx2xt/Dlu7uZggc=";

    hash = "sha256-6RURdPU1k3GaQAgA1LMQ0NhSGBEpOEJBPvk2QjLdoHo=";

    hash_deb_amd64 = "sha256-0FB1gTbsjqFRy0ocE0w5ACtD9kSJ5AMnxg+qBxqCulc=";

    version = "120.0.6099.129";

  };