Merge pull request #194310 from lilyinstarlight/pkg/curl-impersonate

  couchdb = handleTest ./couchdb.nix {};

  cri-o = handleTestOn ["aarch64-linux" "x86_64-linux"] ./cri-o.nix {};

  cups-pdf = handleTest ./cups-pdf.nix {};

  curl-impersonate = handleTest ./curl-impersonate.nix {};

  custom-ca = handleTest ./custom-ca.nix {};

  croc = handleTest ./croc.nix {};

  darling = handleTest ./darling.nix {};

/*

  Test suite for curl-impersonate

  Abstract:

    Uses the test suite from the curl-impersonate source repo which:

    1. Performs requests with libcurl and captures the TLS client-hello

       packets with tcpdump to compare against known-good signatures

    2. Spins up an nghttpd2 server to test client HTTP/2 headers against

       known-good headers

    See https://github.com/lwthiker/curl-impersonate/tree/main/tests/signatures

    for details.

  Notes:

    - We need to have our own web server running because the tests expect to be able

      to hit domains like wikipedia.org and the sandbox has no internet

    - We need to be able to do (verifying) TLS handshakes without internet access.

      We do that by creating a trusted CA and issuing a cert that includes

      all of the test domains as subject-alternative names and then spoofs the

      hostnames in /etc/hosts.

*/

import ./make-test-python.nix ({ pkgs, lib, ... }: let

  # Update with domains in TestImpersonate.TEST_URLS if needed from:

  # https://github.com/lwthiker/curl-impersonate/blob/main/tests/test_impersonate.py

  domains = [

    "www.wikimedia.org"

    "www.wikipedia.org"

    "www.mozilla.org"

    "www.apache.org"

    "www.kernel.org"

    "git-scm.com"

  ];

  tls-certs = let

    # Configure CA with X.509 v3 extensions that would be trusted by curl

    ca-cert-conf = pkgs.writeText "curl-impersonate-ca.cnf" ''

      basicConstraints = critical, CA:TRUE

      subjectKeyIdentifier = hash

      authorityKeyIdentifier = keyid:always, issuer:always

      keyUsage = critical, cRLSign, digitalSignature, keyCertSign

    '';

    # Configure leaf certificate with X.509 v3 extensions that would be trusted

    # by curl and set subject-alternative names for test domains

    tls-cert-conf = pkgs.writeText "curl-impersonate-tls.cnf" ''

      basicConstraints = critical, CA:FALSE

      subjectKeyIdentifier = hash

      authorityKeyIdentifier = keyid:always, issuer:always

      keyUsage = critical, nonRepudiation, digitalSignature, keyEncipherment, keyAgreement

      extendedKeyUsage = critical, serverAuth

      subjectAltName = @alt_names

      [alt_names]

      ${lib.concatStringsSep "\n" (lib.imap0 (idx: domain: "DNS.${toString idx} = ${domain}") domains)}

    '';

  in pkgs.runCommand "curl-impersonate-test-certs" {

    nativeBuildInputs = [ pkgs.openssl ];

  } ''

    # create CA certificate and key

    openssl req -newkey rsa:4096 -keyout ca-key.pem -out ca-csr.pem -nodes -subj '/CN=curl-impersonate-ca.nixos.test'

    openssl x509 -req -sha512 -in ca-csr.pem -key ca-key.pem -out ca.pem -extfile ${ca-cert-conf} -days 36500

    openssl x509 -in ca.pem -text

    # create server certificate and key

    openssl req -newkey rsa:4096 -keyout key.pem -out csr.pem -nodes -subj '/CN=curl-impersonate.nixos.test'

    openssl x509 -req -sha512 -in csr.pem -CA ca.pem -CAkey ca-key.pem -CAcreateserial -out cert.pem -extfile ${tls-cert-conf} -days 36500

    openssl x509 -in cert.pem -text

    # output CA cert and server cert and key

    mkdir -p $out

    cp key.pem cert.pem ca.pem $out

  '';

  # Test script

  curl-impersonate-test = let

    # Build miniature libcurl client used by test driver

    minicurl = pkgs.runCommandCC "minicurl" {

      buildInputs = [ pkgs.curl ];

    } ''

      mkdir -p $out/bin

      $CC -Wall -Werror -o $out/bin/minicurl ${pkgs.curl-impersonate.src}/tests/minicurl.c `curl-config --libs`

    '';

  in pkgs.writeShellScript "curl-impersonate-test" ''

    set -euxo pipefail

    # Test driver requirements

    export PATH="${with pkgs; lib.makeBinPath [

      bash

      coreutils

      python3Packages.pytest

      nghttp2

      tcpdump

    ]}"

    export PYTHONPATH="${with pkgs.python3Packages; makePythonPath [

      pyyaml

      pytest-asyncio

      dpkt

    ]}"

    # Prepare test root prefix

    mkdir -p usr/{bin,lib}

    cp -rs ${pkgs.curl-impersonate}/* ${minicurl}/* usr/

    cp -r ${pkgs.curl-impersonate.src}/tests ./

    # Run tests

    cd tests

    pytest . --install-dir ../usr --capture-interface eth1

  '';

in {

  name = "curl-impersonate";

  meta = with lib.maintainers; {

    maintainers = [ lilyinstarlight ];

  };

  nodes = {

    web = { nodes, pkgs, lib, config, ... }: {

      networking.firewall.allowedTCPPorts = [ 80 443 ];

      services = {

        nginx = {

          enable = true;

          virtualHosts."curl-impersonate.nixos.test" = {

            default = true;

            addSSL = true;

            sslCertificate = "${tls-certs}/cert.pem";

            sslCertificateKey = "${tls-certs}/key.pem";

          };

        };

      };

    };

    curl = { nodes, pkgs, lib, config, ... }: {

      networking.extraHosts = lib.concatStringsSep "\n" (map (domain: "${nodes.web.networking.primaryIPAddress}  ${domain}") domains);

      security.pki.certificateFiles = [ "${tls-certs}/ca.pem" ];

    };

  };

  testScript = { nodes, ... }: ''

    start_all()

    with subtest("Wait for network"):

        web.wait_for_unit("network-online.target")

        curl.wait_for_unit("network-online.target")

    with subtest("Wait for web server"):

        web.wait_for_unit("nginx.service")

        web.wait_for_open_port(443)

    with subtest("Run curl-impersonate tests"):

        curl.succeed("${curl-impersonate-test}")

  '';

})

diff --git a/Makefile.in b/Makefile.in

index 877c54f..3e39ed1 100644

--- a/Makefile.in

+++ b/Makefile.in

@@ -209,6 +209,8 @@ $(NSS_VERSION).tar.gz:

 $(nss_static_libs): $(NSS_VERSION).tar.gz

 	tar xf $(NSS_VERSION).tar.gz

+	sed -i -e "1s@#!/usr/bin/env bash@#!$$(type -p bash)@" $(NSS_VERSION)/nss/build.sh

+	sed -i -e "s@/usr/bin/env grep@$$(type -p grep)@" $(NSS_VERSION)/nss/coreconf/config.gypi

 ifeq ($(host),$(build))

 	# Native build, use NSS' build script.

#TODO: It should be possible to build this from source, but it's currently a lot faster to just package the binaries.

{ lib, stdenv, fetchzip, zlib, autoPatchelfHook }:

stdenv.mkDerivation rec {

  pname = "curl-impersonate-bin";

  version = "v0.5.3";

  src = fetchzip {

    url = "https://github.com/lwthiker/curl-impersonate/releases/download/${version}/curl-impersonate-${version}.x86_64-linux-gnu.tar.gz";

    sha256 = "sha256-+cH1swAIadIrWG9anzf0dcW6qyBjcKsUHFWdv75F49g=";

    stripRoot = false;

{ lib

, stdenv

, fetchFromGitHub

, fetchpatch

, callPackage

, buildGoModule

, installShellFiles

, symlinkJoin

, zlib

, sqlite

, cmake

, python3

, ninja

, perl

, autoconf

, automake

, libtool

, darwin

, cacert

, unzip

, go

, p11-kit

, nixosTests

}:

let

  makeCurlImpersonate = { name, target }: stdenv.mkDerivation rec {

    pname = "curl-impersonate-${name}";

    version = "0.5.4";

    src = fetchFromGitHub {

      owner = "lwthiker";

      repo = "curl-impersonate";

      rev = "v${version}";

      hash = "sha256-LBGWFal2szqgURIBCLB84kHWpdpt5quvBBZu6buGj2A=";

    };

  nativeBuildInputs = [ autoPatchelfHook zlib ];

    patches = [

      # Fix shebangs in the NSS build script

      # (can't just patchShebangs since makefile unpacks it)

      ./curl-impersonate-0.5.2-fix-shebangs.patch

    ];

    strictDeps = true;

    nativeBuildInputs = lib.optionals stdenv.isDarwin [

      # Must come first so that it shadows the 'libtool' command but leaves 'libtoolize'

      darwin.cctools

    ] ++ [

      installShellFiles

      cmake

      python3

      python3.pkgs.gyp

      ninja

      perl

      autoconf

      automake

      libtool

      unzip

      go

    ];

    buildInputs = [

      zlib

      sqlite

    ];

    configureFlags = [

      "--with-ca-bundle=${if stdenv.isDarwin then "/etc/ssl/cert.pem" else "/etc/ssl/certs/ca-certificates.crt"}"

      "--with-ca-path=${cacert}/etc/ssl/certs"

    ];

    buildFlags = [ "${target}-build" ];

    checkTarget = "${target}-checkbuild";

    installTargets = [ "${target}-install" ];

    doCheck = true;

    dontUseCmakeConfigure = true;

    dontUseNinjaBuild = true;

    dontUseNinjaInstall = true;

    dontUseNinjaCheck = true;

    postUnpack = lib.concatStringsSep "\n" (lib.mapAttrsToList (name: dep: "ln -sT ${dep.outPath} source/${name}") (lib.filterAttrs (n: v: v ? outPath) passthru.deps));

    preConfigure = ''

      export GOCACHE=$TMPDIR/go-cache

      export GOPATH=$TMPDIR/go

      export GOPROXY=file://${passthru.boringssl-go-modules}

      export GOSUMDB=off

      # Need to get value of $out for this flag

      configureFlagsArray+=("--with-libnssckbi=$out/lib")

    '';

    postInstall = ''

      # Remove vestigial *-config script

      rm $out/bin/curl-impersonate-${name}-config

      # Patch all shebangs of installed scripts

      patchShebangs $out/bin

      # Build and install completions for each curl binary

      # Patch in correct binary name and alias it to all scripts

      perl curl-*/scripts/completion.pl --curl $out/bin/curl-impersonate-${name} --shell zsh >$TMPDIR/curl-impersonate-${name}.zsh

      substituteInPlace $TMPDIR/curl-impersonate-${name}.zsh \

        --replace \

          '#compdef curl' \

          "#compdef curl-impersonate-${name}$(find $out/bin -name 'curl_*' -printf ' %f=curl-impersonate-${name}')"

      perl curl-*/scripts/completion.pl --curl $out/bin/curl-impersonate-${name} --shell fish >$TMPDIR/curl-impersonate-${name}.fish

      substituteInPlace $TMPDIR/curl-impersonate-${name}.fish \

        --replace \

          '--command curl' \

          "--command curl-impersonate-${name}$(find $out/bin -name 'curl_*' -printf ' --command %f')"

  installPhase = ''

    mkdir -p $out/bin

    cp * $out/bin

      # Install zsh and fish completions

      installShellCompletion $TMPDIR/curl-impersonate-${name}.{zsh,fish}

    '';

    preFixup = let

      libext = stdenv.hostPlatform.extensions.sharedLibrary;

    in ''

      # If libnssckbi.so is needed, link libnssckbi.so without needing nss in closure

      if grep -F nssckbi $out/lib/libcurl-impersonate-*${libext} &>/dev/null; then

        # NOTE: "p11-kit-trust" always ends in ".so" even when on darwin

        ln -s ${p11-kit}/lib/pkcs11/p11-kit-trust.so $out/lib/libnssckbi${libext}

        ${lib.optionalString stdenv.isLinux "patchelf --add-needed libnssckbi${libext} $out/lib/libcurl-impersonate-*${libext}"}

      fi

    '';

    disallowedReferences = [ go ];

    passthru = {

      deps = callPackage ./deps.nix {};

      boringssl-go-modules = (buildGoModule {

        inherit (passthru.deps."boringssl.zip") name;

        src = passthru.deps."boringssl.zip";

        vendorHash = "sha256-ISmRdumckvSu7hBXrjvs5ZApShDiGLdD3T5B0fJ1x2Q=";

        nativeBuildInputs = [ unzip ];

        proxyVendor = true;

      }).go-modules;

    };

    meta = with lib; {

    description = "curl-impersonate: A special build of curl that can impersonate Chrome & Firefox ";

      description = "A special build of curl that can impersonate Chrome & Firefox";

      homepage = "https://github.com/lwthiker/curl-impersonate";

      license = with licenses; [ curl mit ];

    maintainers = with maintainers; [ deliciouslytyped ];

    platforms = platforms.linux; #TODO I'm unsure about the restrictions here, feel free to expand the platforms it if it works elsewhere.

      maintainers = with maintainers; [ deliciouslytyped lilyinstarlight ];

      platforms = platforms.unix;

      knownVulnerabilities = [

        "CVE-2023-32001"  # fopen TOCTOU race condition - https://curl.se/docs/CVE-2023-32001.html

        "CVE-2022-43551"  # HSTS bypass - https://curl.se/docs/CVE-2022-43551.html

        "CVE-2022-42916"  # HSTS bypass - https://curl.se/docs/CVE-2022-42916.html

      ];

    };

  };

in

symlinkJoin rec {

  pname = "curl-impersonate";

  inherit (passthru.curl-impersonate-ff) version meta;

  name = "${pname}-${version}";

  paths = [

    passthru.curl-impersonate-ff

    passthru.curl-impersonate-chrome

  ];

  passthru = {

    curl-impersonate-ff = makeCurlImpersonate { name = "ff"; target = "firefox"; };

    curl-impersonate-chrome = makeCurlImpersonate { name = "chrome"; target = "chrome"; };

    updateScript = ./update.sh;

    inherit (passthru.curl-impersonate-ff) src;

    tests = { inherit (nixosTests) curl-impersonate; };

  };

}

# Generated by update.sh

{ fetchurl }:

{

  "curl-7.84.0.tar.xz" = fetchurl {

    url = "https://curl.se/download/curl-7.84.0.tar.xz";

    hash = "sha256-LRGLQ/VHv+W66AbY1HtOWW6lslpsHwgK70n7zYF8Xbg=";

  };

  "brotli-1.0.9.tar.gz" = fetchurl {

    url = "https://github.com/google/brotli/archive/refs/tags/v1.0.9.tar.gz";

    hash = "sha256-+ejYHQQFumbRgVKa9CozVPg4yTkJX/mZMNpqqc32/kY=";

  };

  "nss-3.87.tar.gz" = fetchurl {

    url = "https://ftp.mozilla.org/pub/security/nss/releases/NSS_3_87_RTM/src/nss-3.87-with-nspr-4.35.tar.gz";

    hash = "sha256-63DqC1jc5pqkkOnp/s0TKn1kTh2j1jHhYzdqDcwRoCI=";

  };

  "boringssl.zip" = fetchurl {

    url = "https://github.com/google/boringssl/archive/3a667d10e94186fd503966f5638e134fe9fb4080.zip";

    hash = "sha256-HsDIkd1x5IH49fUF07dJaabMIMsQygW+NI7GneULpA8=";

  };

  "nghttp2-1.46.0.tar.bz2" = fetchurl {

    url = "https://github.com/nghttp2/nghttp2/releases/download/v1.46.0/nghttp2-1.46.0.tar.bz2";

    hash = "sha256-moKXjIcAcbdp8n0riBkct3/clFpRwdaFx/YafhP8Ryk=";

  };

}