gfal2-util, python3Packages.gfal2-util: init at 1.8.1

{ python3Packages }:

with python3Packages; toPythonApplication gfal2-util

{ lib

, buildPythonPackage

, callPackage

, fetchFromGitHub

, runCommandLocal

  # Build inputs

, gfal2-python

  # For tests

, xrootd # pkgs.xrootd

}:

(buildPythonPackage rec {

  pname = "gfal2-util";

  version = "1.8.1";

  src = fetchFromGitHub {

    owner = "cern-fts";

    repo = "gfal2-util";

    rev = "v${version}";

    hash = "sha256-3JbJgKD17aYkrB/aaww7IQU8fLFrTCh868KWlLPxmlk=";

  };

  # Replace the ad-hoc python executable finding

  # and change the shebangs from `#!/bin/sh` to `#!/usr/bin/env python`

  # for fixup phase to work correctly.

  postPatch = ''

    for script in src/gfal-*; do

      patch "$script" ${./gfal-util-script.patch}

    done

  '';

  propagatedBuildInputs = [

    gfal2-python

  ];

  pythonImportsCheck = [

    "gfal2_util"

  ];

  meta = with lib; {

    description = "CLI for gfal2";

    homepage = "https://github.com/cern-fts/gfal2-utils";

    license = licenses.asl20;

    maintainers = with maintainers; [ ShamrockLee ];

  };

}).overrideAttrs (finalAttrs: previousAttrs: lib.recursiveUpdate previousAttrs {

  passthru = {

    inherit (gfal2-python) gfal2;

    fetchGfal2 = lib.makeOverridable (callPackage ./fetchgfal2.nix { gfal2-util = finalAttrs.finalPackage; });

    # With these functionality tests, it should be safe to merge version bumps once all the tests are passed.

    tests =

      let

        # Use the the bin output hash of gfal2-util as version to ensure that

        # the test gets rebuild everytime gfal2-util gets rebuild

        versionFODTests = finalAttrs.version + "-" + lib.substring (lib.stringLength builtins.storeDir + 1) 32 "${self}";

        self = finalAttrs.finalPackage;

      in

      lib.optionalAttrs gfal2-python.gfal2.enablePluginStatus.xrootd (

        let

          # Test against a real-world dataset from CERN Open Data

          # borrowed from `xrootd.tests`.

          urlTestFile = xrootd.tests.test-xrdcp.url;

          hashTestFile = xrootd.tests.test-xrdcp.outputHash;

          urlTestDir = dirOf urlTestFile;

        in

        {

          test-copy-file-xrootd = finalAttrs.passthru.fetchGfal2 {

            url = urlTestFile;

            hash = hashTestFile;

            extraGfalCopyFlags = [ "--verbose" ];

            pname = "gfal2-util-test-copy-file-xrootd";

            version = versionFODTests;

            allowSubstitutes = false;

          };

          test-copy-dir-xrootd = finalAttrs.passthru.fetchGfal2 {

            url = urlTestDir;

            hash = "sha256-vOahIhvx1oE9sfkqANMGUvGeLHS737wyfYWo4rkvrxw=";

            recursive = true;

            extraGfalCopyFlags = [ "--verbose" ];

            pname = "gfal2-util-test-copy-dir-xrootd";

            version = versionFODTests;

            allowSubstitutes = false;

          };

          test-ls-dir-xrootd = (runCommandLocal "test-gfal2-util-ls-dir-xrootd" { } ''

            set -eu -o pipefail

            gfal-ls "$url" | grep "$baseNameExpected" | tee "$out"

          '').overrideAttrs (finalAttrs: previousAttrs: {

            pname = previousAttrs.name;

            version = versionFODTests;

            name = "${finalAttrs.pname}-${finalAttrs.version}";

            nativeBuildInputs = [ self ];

            url = urlTestDir;

            baseNameExpected = baseNameOf urlTestFile;

            outputHashMode = "flat";

            outputHashAlgo = "sha256";

            outputHash = builtins.hashString finalAttrs.outputHashAlgo (finalAttrs.baseNameExpected + "\n");

          });

        }

      );

  };

})

{ lib

, runCommandLocal

, gfal2-util

}:

# `url` and `urls` should only be overriden via `<pkg>.override`, but not `<pkg>.overrideAttrs`.

{ name ? ""

, pname ? ""

, version ? ""

, urls ? [ ]

, url ? if urls == [ ] then abort "Expect either non-empty `urls` or `url`" else lib.head urls

, hash ? lib.fakeHash

, recursive ? false

, intermediateDestUrls ? [ ]

, extraGfalCopyFlags ? [ ]

, allowSubstitutes ? true

}:

(runCommandLocal name { } ''

  for u in "''${urls[@]}"; do

    gfal-copy "''${gfalCopyFlags[@]}" "$u" "''${intermediateDestUrls[@]}" "$out"

    ret="$?"

    (( ret )) && break

  done

  if (( ret )); then

    echo "gfal-copy failed trying to download from any of the urls" >&2

    exit "$ret"

  fi

'').overrideAttrs (finalAttrs: previousAttrs:

{

  __structuredAttrs = true;

  inherit allowSubstitutes;

  nativeBuildInputs = [ gfal2-util ];

  outputHashAlgo = null;

  outputHashMode = if finalAttrs.recursive then "recursive" else "flat";

  outputHash = hash;

  inherit url;

  urls = if urls == [ ] then lib.singleton url else urls;

  gfalCopyFlags = extraGfalCopyFlags

  ++ lib.optional finalAttrs.recursive "--recursive"

  ;

  inherit recursive intermediateDestUrls;

} // (if (pname != "" && version != "") then {

  inherit pname version;

  name = "${finalAttrs.pname}-${finalAttrs.version}";

} else {

  name = if (name != "") then name else (baseNameOf url);

}))

--- a/gfal-SCRIPT_NAME	2022-11-17 00:00:00.000000000 +0000

+++ b/gfal-SCRIPT_NAME	2022-11-17 00:00:00.000000000 +0000

@@ -1,4 +1,4 @@

-#!/bin/sh

+#!/usr/bin/env python3

 # -*- coding: utf-8 -*-

 #

 # Copyright (c) 2022 CERN

@@ -15,10 +15,2 @@

 # See the License for the specific language governing permissions and

 # limitations under the License.

-

-# Execute script content with first python interpreter found:

-# * GFAL_PYTHONBIN environment variable

-# * python on the PATH if import gfal2, gfal2_util succeeds

-# * python3 on the PATH if import gfal2, gfal2_util succeeds

-# * python2 on the PATH if import gfal2, gfal2_util succeeds

-# * /usr/bin/python

-"exec" "$(  check_interpreter() { unalias $1 2> /dev/null; unset $1; GFAL_PYTHONBIN=$(command -v $1); [ $GFAL_PYTHONBIN ] && $GFAL_PYTHONBIN -c 'import gfal2, gfal2_util' > /dev/null 2>&1 && { echo $GFAL_PYTHONBIN; unset GFAL_PYTHONBIN; }; }; [ $GFAL_PYTHONBIN ] && echo $GFAL_PYTHONBIN || check_interpreter python || check_interpreter python3 || check_interpreter python2 || echo /usr/bin/python )" "-u" "-Wignore" "$0" "$@"

  github-copilot-cli = callPackage ../tools/misc/github-copilot-cli { };

  # This is to workaround gfal2-python broken against Python 3.12 or later.

  # TODO: Remove these lines after solving the breakage.

  gfal2-util = callPackage ../by-name/gf/gfal2-util/package.nix (lib.optionalAttrs python3Packages.gfal2-python.meta.broken {

    python3Packages = python311Packages;

  });

  gfshare = callPackage ../tools/security/gfshare { };

  gh-actions-cache = callPackage ../tools/misc/gh-actions-cache { };