Unverified Commit 8ddddcd3 authored by Sandro Jäckel's avatar Sandro Jäckel Committed by GitHub
Browse files

pgadmin: use new yarn-berry_3 fetcher (#402519)

parents 86748315 f3cef735

pkgs/tools/admin/pgadmin/default.nix

+31 −25
Original line number Diff line number Diff line
@@ -2,14 +2,18 @@ 
  lib,

  python3,

  fetchFromGitHub,

  fetchYarnDeps,

  zlib,

  nixosTests,

  postgresqlTestHook,

  postgresql,

  yarn,

  fixup-yarn-lock,

  yarn-berry_3,

  nodejs,

  autoconf,

  automake,

  libtool,

  libpng,

  nasm,

  pkg-config,

  stdenv,

  server-mode ? true,

}:
@@ -17,7 +21,7 @@ 
let

  pname = "pgadmin";

  version = "9.2";

  yarnHash = "sha256-nFYzaMRASkorEJC5UgLZjDY99ptwPqOMxOomhY/CY5k=";

  yarnHash = "sha256-I7Eua6MkZR7l4Ks1Cyz0AAXLuayuzZHuk+xtn8Zu8UI=";



  src = fetchFromGitHub {

    owner = "pgadmin-org";
@@ -30,11 +34,6 @@ let 
  # this also makes potential future overrides easier

  pythonPackages = python3.pkgs.overrideScope (final: prev: { });



  offlineCache = fetchYarnDeps {

    yarnLock = ./yarn.lock;

    hash = yarnHash;

  };



  # don't bother to test kerberos authentication

  # skip tests on macOS which fail due to an error in keyring, see https://github.com/NixOS/nixpkgs/issues/281214

  skippedTests = builtins.concatStringsSep "," (
@@ -51,6 +50,11 @@ in 
pythonPackages.buildPythonApplication rec {

  inherit pname version src;



  offlineCache = yarn-berry_3.fetchYarnBerryDeps {

    src = src + "/web";

    hash = yarnHash;

  };



  # from Dockerfile

  CPPFLAGS = "-DPNG_ARM_NEON_OPT=0";
@@ -76,9 +80,6 @@ pythonPackages.buildPythonApplication rec { 
    sed 's|==|>=|g' -i requirements.txt

    # fix extra_require error with "*" in match

    sed 's|*|0|g' -i requirements.txt

    # remove packageManager from package.json so we can work without corepack

    substituteInPlace web/package.json \

      --replace-fail "\"packageManager\": \"yarn@3.8.7\"" "\"\": \"\""

    substituteInPlace pkg/pip/setup_pip.py \

      --replace-fail "req = req.replace('psycopg[c]', 'psycopg[binary]')" "req = req"

    ${lib.optionalString (!server-mode) ''
@@ -87,6 +88,8 @@ pythonPackages.buildPythonApplication rec { 
    ''}

  '';



  dontYarnBerryInstallDeps = true;



  preBuild = ''

    # Adapted from pkg/pip/build.sh

    echo Creating required directories...
@@ -108,18 +111,12 @@ pythonPackages.buildPythonApplication rec { 
    done

    cd ../



    # mkYarnModules and mkYarnPackage have problems running the webpacker

    echo Building the web frontend...

    cd web

    export HOME="$TMPDIR"

    yarn config --offline set yarn-offline-mirror "${offlineCache}"

    # replace with converted yarn.lock file

    rm yarn.lock

    cp ${./yarn.lock} yarn.lock

    chmod +w yarn.lock

    fixup-yarn-lock yarn.lock

    yarn install --offline --frozen-lockfile --ignore-platform --ignore-scripts --no-progress --non-interactive

    patchShebangs node_modules/

    (

    export LD=$CC # https://github.com/imagemin/optipng-bin/issues/108

    yarnBerryConfigHook

    )

    yarn webpacker

    cp -r * ../pip-build/pgadmin4

    # save some disk space
@@ -145,13 +142,23 @@ pythonPackages.buildPythonApplication rec { 
    cython

    pip

    sphinx

    yarn

    fixup-yarn-lock

    yarn-berry_3

    yarn-berry_3.yarnBerryConfigHook

    nodejs



    # for building mozjpeg2

    autoconf

    automake

    libtool

    nasm

    pkg-config

  ];



  buildInputs = [

    zlib

    pythonPackages.wheel

    # for mozjpeg2

    libpng

  ];



  propagatedBuildInputs = with pythonPackages; [
@@ -242,7 +249,6 @@ pythonPackages.buildPythonApplication rec { 
    python regression/runtests.py --pkg browser --exclude ${skippedTests}



    ## Reverse engineered SQL test ##



    python regression/runtests.py --pkg resql



    runHook postCheck

pkgs/tools/admin/pgadmin/update.sh

+16 −57
Original line number Diff line number Diff line 
#!/usr/bin/env nix-shell

#!nix-shell -i bash -p curl wget jq yq common-updater-scripts prefetch-yarn-deps yarn-lock-converter

#!nix-shell -i bash -p curl wget jq common-updater-scripts yarn-berry_3 yarn-berry_3.yarn-berry-fetcher



set -eu -o pipefail



TMPDIR=/tmp/pgadmin-update-script



################################################################

#         This script will update pgadmin4 in nixpkgs          #

# Due to recent changes upstream, we will need to convert the  #

#             `yarn.lock` file back to version 1.              #

#   This isn't trivially done and relies on 3rd party tools    #

#       and a hand-written converter (in this script).         #

# Also, the converter cannot check for `github` repos in the   #

#   `yarn.lock` file, which this script will add automatically #

################################################################



cleanup() {

  if [ -e $TMPDIR/.done ]

  then
@@ -52,55 +42,24 @@ wget -c $url 
tar -xzf "pgadmin4-$newest_version.tar.gz"

cd "pgadmin4-$newest_version/web"



printf "Will now convert the v2 lockfile. This will download the npm packages to get the metadata.\n"

printf "Please note: This will take some time! For details, see the logfile ${TMPDIR}/update.log\n"

yarn-lock-converter -i yarn.lock -o yarn_v1.lock --cache .cache > $TMPDIR/update.log

printf "Conversion done\n"



printf "Will now do some regex substitution post-processing\n"

sed -i -E "s|(.), |\1\", \"|g" yarn_v1.lock

sed -i -E "s|npm:||g" yarn_v1.lock

printf "Substituion done\n"



printf "Will now add missing github packages back to the v1 yarn.lock file\n"

# remove header

tail +8 yarn.lock > yarn_mod.lock

LENGTH=$(yq '. | with_entries(select(.value.resolution | contains("github"))) | keys | length' yarn_mod.lock)

for i in $(seq 0 $(($LENGTH-1)));

do

  ENTRY=$(yq ". | with_entries(select(.value.resolution | contains(\"github\"))) | keys | .[$i]" yarn_mod.lock)

  URL=$(echo $ENTRY | cut -d "@" -f 2)

  VERSION=$(yq ".$ENTRY.version" yarn_mod.lock)

  LENGTH_DEP=$(yq ".$ENTRY.dependencies | keys | length" yarn_mod.lock)

  echo "$ENTRY:" >> adendum.lock

  echo "  version $VERSION" >> adendum.lock

  echo "  resolved \"$URL" >> adendum.lock

  echo "  dependencies:" >> adendum.lock



  for j in $(seq 0 $(($LENGTH_DEP-1)));

  do

    DEPENDENCY_KEY=$(yq ".$ENTRY.dependencies | keys | .[$j]" yarn_mod.lock)

    DEPENDENCY_VALUE=$(yq ".$ENTRY.dependencies.$DEPENDENCY_KEY" yarn_mod.lock)

    # remove '"'

    DEPENDENCY_KEY=${DEPENDENCY_KEY//\"}

    echo "    \"$DEPENDENCY_KEY\" $DEPENDENCY_VALUE" >> adendum.lock

  done

done



echo "" >> yarn_v1.lock

cat adendum.lock >> yarn_v1.lock

printf "Done\n"



rm yarn.lock

mv yarn_v1.lock yarn.lock



printf "Will now generate the hash. This will download the packages to the nix store and also take some time\n"

YARN_HASH=$(prefetch-yarn-deps yarn.lock)

YARN_HASH=$(nix hash to-sri --type sha256 "$YARN_HASH")

yarn-berry-fetcher missing-hashes yarn.lock

if [[ -f missing-hashes.json ]]; then

  YARN_HASH=$(yarn-berry-fetcher prefetch yarn.lock missing-hashes.json)

else

  YARN_HASH=$(yarn-berry-fetcher prefetch yarn.lock)

fi

printf "Done\n"



if [[ -f missing-hashes.json ]]; then

  if [[ ! -f "$nixpkgs/pkgs/tools/admin/pgadmin/missing-hashes.json" ]]; then

    printf "PLEASE NOTE: FIRST TIME OF FINDING MISSING HASHES!"

    printf "Please add \"missingHashes = ./missing-hashes.json\" to pgadmin derivation"

  fi

  printf "Copy files to nixpkgs\n"

cp yarn.lock "$nixpkgs/pkgs/tools/admin/pgadmin/"

  cp missing-hashes.json "$nixpkgs/pkgs/tools/admin/pgadmin/"

fi



printf "Done\n"

popd

pkgs/tools/admin/pgadmin/yarn.lock

deleted100644 → 0
+0 −11858

File deleted.

Preview size limit exceeded, changes collapsed.