Commit 67b5c21b authored by phaer's avatar phaer
Browse files

playwright: Use pre-built browsers, update them... 

via update.sh. This lets us support playwright with browsers
other than chromium on linux. Building them from source would
be one step further.
parent e9e4fb40

pkgs/development/python-modules/playwright/update.sh

+47 −7
Original line number Diff line number Diff line 
#!/usr/bin/env nix-shell

#!nix-shell -i bash -p curl gnused common-updater-scripts jq prefetch-npm-deps

#!nix-shell -i bash -p curl gnused common-updater-scripts jq prefetch-npm-deps unzip

set -euo pipefail



root="$(dirname "$(readlink -f "$0")")"
@@ -11,18 +11,60 @@ version=$(curl ${GITHUB_TOKEN:+" -u \":$GITHUB_TOKEN\""} -s https://api.github.c 
setup_py_url="https://github.com/microsoft/playwright-python/raw/v${version}/setup.py"

driver_version=$(curl -Ls "$setup_py_url" | grep '^driver_version =' | grep -Eo '[0-9]+\.[0-9]+\.[0-9]+')



# TODO: skip if update-source-version reported the same version

update-source-version playwright-driver "$driver_version"

update-source-version python3Packages.playwright "$version"



# Update package-lock.json files for all npm deps that are built in playwright

# TODO: skip if update-source-version reported the same version

driver_file="$root/../../web/playwright/driver.nix"

playwright_dir="$root/../../web/playwright"

driver_file="$playwright_dir/driver.nix"

repo_url_prefix="https://github.com/microsoft/playwright/raw"

temp_dir=$(mktemp -d)



temp_dir=$(mktemp -d)

trap 'rm -rf "$temp_dir"' EXIT







# update binaries of browsers, used by playwright.

replace_sha() {

  sed -i "s|$2 = \".\{44,52\}\"|$2 = \"$3\"|" "$1"

}



prefetch_browser() {

    nix store prefetch-file --json --hash-type sha256 --unpack "$1" | jq -r .hash

}



update_browser() {

    name="$1"

    suffix="$2"

    arm64_suffix="${3:-$2-arm64}"

    revision="$(jq -r ".browsers.$name.revision" "$playwright_dir/browsers.json")"

    replace_sha "$playwright_dir/$name.nix" "x86_64-linux" \

        "$(prefetch_browser "https://playwright.azureedge.net/builds/$name/$revision/$name-$suffix.zip")"

    replace_sha "$playwright_dir/$name.nix" "aarch64-linux" \

        "$(prefetch_browser "https://playwright.azureedge.net/builds/$name/$revision/$name-$arm64_suffix.zip")"

}



curl -fsSl \

    "https://raw.githubusercontent.com/microsoft/playwright/v${driver_version}/packages/playwright-core/browsers.json" \

    | jq '

      .comment = "This file is kept up to date via update.sh"

      | .browsers |= (

        [.[]

          | select(.installByDefault) | del(.installByDefault)]

          | map({(.name): . | del(.name)})

          | add

      )

    ' > "$playwright_dir/browsers.json"



# We currently use Chromium from nixpkgs, so we don't need to download it here

# Likewise, darwin can be ignored here atm as we are using an impure install anyway.

update_browser "firefox" "ubuntu-22.04"

update_browser "webkit" "ubuntu-22.04"

update_browser "ffmpeg" "linux"





# Update package-lock.json files for all npm deps that are built in playwright



# Function to download `package-lock.json` for a given source path and update hash

update_hash() {

    local source_root_path="$1"
@@ -30,7 +72,6 @@ update_hash() { 


    # Formulate download URL

    local download_url="${repo_url_prefix}/v${driver_version}${source_root_path}/package-lock.json"



    # Download package-lock.json to temporary directory

    curl -fsSL -o "${temp_dir}/package-lock.json" "$download_url"
@@ -45,7 +86,6 @@ update_hash() { 
while IFS= read -r source_root_line; do

    [[ "$source_root_line" =~ sourceRoot ]] || continue

    source_root_path=$(echo "$source_root_line" | sed -e 's/^.*"${src.name}\(.*\)";.*$/\1/')



    # Extract the current npmDepsHash for this sourceRoot

    existing_hash=$(grep -A1 "$source_root_line" "$driver_file" | grep 'npmDepsHash' | sed -e 's/^.*npmDepsHash = "\(.*\)";$/\1/')

pkgs/development/web/playwright/browsers.json

0 → 100644
+28 −0
Original line number Diff line number Diff line 
{

  "comment": "This file is kept up to date via update.sh",

  "browsers": {

    "chromium": {

      "revision": "1134",

      "browserVersion": "129.0.6668.29"

    },

    "firefox": {

      "revision": "1463",

      "browserVersion": "130.0"

    },

    "webkit": {

      "revision": "2070",

      "revisionOverrides": {

        "mac10.14": "1446",

        "mac10.15": "1616",

        "mac11": "1816",

        "mac11-arm64": "1816",

        "mac12": "2009",

        "mac12-arm64": "2009"

      },

      "browserVersion": "18.0"

    },

    "ffmpeg": {

      "revision": "1010"

    }

  }

}

pkgs/development/web/playwright/chromium.nix

0 → 100644
+29 −0
Original line number Diff line number Diff line 
{

  runCommand,

  makeWrapper,

  makeFontsConf,

  chromium,

  ...

}:

let

  fontconfig = makeFontsConf {

    fontDirectories = [ ];

  };

in

runCommand "playwright-chromium"

  {

    nativeBuildInputs = [

      makeWrapper

    ];

  }

  ''

    mkdir -p $out/chrome-linux



    # See here for the Chrome options:

    # https://github.com/NixOS/nixpkgs/issues/136207#issuecomment-908637738

    # We add --disable-gpu to be able to run in gpu-less environments such

    # as headless nixos test vms.

    makeWrapper ${chromium}/bin/chromium $out/chrome-linux/chrome \

      --set-default SSL_CERT_FILE /etc/ssl/certs/ca-bundle.crt \

      --set-default FONTCONFIG_FILE ${fontconfig}

  ''

pkgs/development/web/playwright/driver.nix

+38 −28
Original line number Diff line number Diff line
@@ -7,6 +7,8 @@ 
  jq,

  nodejs,

  fetchFromGitHub,

  linkFarm,

  callPackage,

  makeFontsConf,

  makeWrapper,

  runCommand,
@@ -16,6 +18,14 @@ let 
  inherit (stdenv.hostPlatform) system;



  throwSystem = throw "Unsupported system: ${system}";

  suffix =

    {

      x86_64-linux = "linux";

      aarch64-linux = "linux-arm64";

      x86_64-darwin = "mac";

      aarch64-darwin = "mac-arm64";

    }

    .${system} or throwSystem;



  version = "1.47.0";
@@ -148,6 +158,7 @@ let 
    '';



    passthru = {

      browsersJSON = (lib.importJSON ./browsers.json).browsers;

      browsers =

        {

          x86_64-linux = browsers-linux { };
@@ -209,36 +220,35 @@ let 
  browsers-linux =

    {

      withChromium ? true,

      withFirefox ? true,

      withWebkit ? true,

      withFfmpeg ? true,

    }:

    let

      fontconfig = makeFontsConf { fontDirectories = [ ]; };

      browsers =

        lib.optionals withChromium [ "chromium" ]

        ++ lib.optionals withFirefox [ "firefox" ]

        ++ lib.optionals withWebkit [ "webkit" ]

        ++ lib.optionals withFfmpeg [ "ffmpeg" ];

    in

    runCommand ("playwright-browsers" + lib.optionalString withChromium "-chromium")

      {

        nativeBuildInputs = [

          makeWrapper

          jq

        ];

      }

    linkFarm "playwright-browsers" (

      lib.listToAttrs (

        map (

          name:

          let

            value = playwright-core.passthru.browsersJSON.${name};

          in

          lib.nameValuePair

            # TODO check platform for revisionOverrides

            "${name}-${value.revision}"

            (

        ''

          BROWSERS_JSON=${playwright-core}/browsers.json

        ''

        + lib.optionalString withChromium ''

          CHROMIUM_REVISION=$(jq -r '.browsers[] | select(.name == "chromium").revision' $BROWSERS_JSON)

          mkdir -p $out/chromium-$CHROMIUM_REVISION/chrome-linux



          # See here for the Chrome options:

          # https://github.com/NixOS/nixpkgs/issues/136207#issuecomment-908637738

          makeWrapper ${chromium}/bin/chromium $out/chromium-$CHROMIUM_REVISION/chrome-linux/chrome \

            --set SSL_CERT_FILE /etc/ssl/certs/ca-bundle.crt \

            --set FONTCONFIG_FILE ${fontconfig}

        ''

        + ''

          FFMPEG_REVISION=$(jq -r '.browsers[] | select(.name == "ffmpeg").revision' $BROWSERS_JSON)

          mkdir -p $out/ffmpeg-$FFMPEG_REVISION

          ln -s ${ffmpeg}/bin/ffmpeg $out/ffmpeg-$FFMPEG_REVISION/ffmpeg-linux

        ''

              callPackage ./${name}.nix {

                inherit suffix system throwSystem;

                inherit (value) revision;

              }

            )

        ) browsers

      )

    );

in

{

pkgs/development/web/playwright/ffmpeg.nix

0 → 100644
+17 −0
Original line number Diff line number Diff line 
{

  fetchzip,

  suffix,

  revision,

  system,

  throwSystem,

}:

fetchzip {

  url = "https://playwright.azureedge.net/builds/ffmpeg/${revision}/ffmpeg-${suffix}.zip";

  stripRoot = false;

  hash =

    {

      x86_64-linux = "sha256-FEm62UvMv0h6Sav93WmbPLw3CW1L1xg4nD26ca5ol38=";

      aarch64-linux = "sha256-jtQ+NS++VHRiKoIV++PIxEnyVnYtVwUyNlSILKSH4A4=";

    }

    .${system} or throwSystem;

}