Commit fc61cff2 authored by Yueh-Shun Li's avatar Yueh-Shun Li
Browse files

gfal2-util, python3Packages.gfal2-util: init at 1.8.1

Add also passthru.fetchGfal2. This fetcher is currently used only by
passthru.tsets, but capable to be used as a domain-specific
real-world fetcher.
parent 9cbe717f
Loading
Loading
Loading
Loading
+2 −0
Original line number Diff line number Diff line
{ python3Packages }:
with python3Packages; toPythonApplication gfal2-util
+103 −0
Original line number Diff line number Diff line
{ lib
, buildPythonPackage
, callPackage
, fetchFromGitHub
, runCommandLocal
  # Build inputs
, gfal2-python
  # For tests
, xrootd # pkgs.xrootd
}:
(buildPythonPackage rec {
  pname = "gfal2-util";
  version = "1.8.1";
  src = fetchFromGitHub {
    owner = "cern-fts";
    repo = "gfal2-util";
    rev = "v${version}";
    hash = "sha256-3JbJgKD17aYkrB/aaww7IQU8fLFrTCh868KWlLPxmlk=";
  };

  # Replace the ad-hoc python executable finding
  # and change the shebangs from `#!/bin/sh` to `#!/usr/bin/env python`
  # for fixup phase to work correctly.
  postPatch = ''
    for script in src/gfal-*; do
      patch "$script" ${./gfal-util-script.patch}
    done
  '';

  propagatedBuildInputs = [
    gfal2-python
  ];

  pythonImportsCheck = [
    "gfal2_util"
  ];

  meta = with lib; {
    description = "CLI for gfal2";
    homepage = "https://github.com/cern-fts/gfal2-utils";
    license = licenses.asl20;
    maintainers = with maintainers; [ ShamrockLee ];
  };
}).overrideAttrs (finalAttrs: previousAttrs: lib.recursiveUpdate previousAttrs {
  passthru = {
    inherit (gfal2-python) gfal2;

    fetchGfal2 = lib.makeOverridable (callPackage ./fetchgfal2.nix { gfal2-util = finalAttrs.finalPackage; });

    # With these functionality tests, it should be safe to merge version bumps once all the tests are passed.
    tests =
      let
        # Use the the bin output hash of gfal2-util as version to ensure that
        # the test gets rebuild everytime gfal2-util gets rebuild
        versionFODTests = finalAttrs.version + "-" + lib.substring (lib.stringLength builtins.storeDir + 1) 32 "${self}";
        self = finalAttrs.finalPackage;
      in
      lib.optionalAttrs gfal2-python.gfal2.enablePluginStatus.xrootd (
        let
          # Test against a real-world dataset from CERN Open Data
          # borrowed from `xrootd.tests`.
          urlTestFile = xrootd.tests.test-xrdcp.url;
          hashTestFile = xrootd.tests.test-xrdcp.outputHash;
          urlTestDir = dirOf urlTestFile;
        in
        {
          test-copy-file-xrootd = finalAttrs.passthru.fetchGfal2 {
            url = urlTestFile;
            hash = hashTestFile;
            extraGfalCopyFlags = [ "--verbose" ];
            pname = "gfal2-util-test-copy-file-xrootd";
            version = versionFODTests;
            allowSubstitutes = false;
          };

          test-copy-dir-xrootd = finalAttrs.passthru.fetchGfal2 {
            url = urlTestDir;
            hash = "sha256-vOahIhvx1oE9sfkqANMGUvGeLHS737wyfYWo4rkvrxw=";
            recursive = true;
            extraGfalCopyFlags = [ "--verbose" ];
            pname = "gfal2-util-test-copy-dir-xrootd";
            version = versionFODTests;
            allowSubstitutes = false;
          };

          test-ls-dir-xrootd = (runCommandLocal "test-gfal2-util-ls-dir-xrootd" { } ''
            set -eu -o pipefail
            gfal-ls "$url" | grep "$baseNameExpected" | tee "$out"
          '').overrideAttrs (finalAttrs: previousAttrs: {
            pname = previousAttrs.name;
            version = versionFODTests;
            name = "${finalAttrs.pname}-${finalAttrs.version}";
            nativeBuildInputs = [ self ];
            url = urlTestDir;
            baseNameExpected = baseNameOf urlTestFile;
            outputHashMode = "flat";
            outputHashAlgo = "sha256";
            outputHash = builtins.hashString finalAttrs.outputHashAlgo (finalAttrs.baseNameExpected + "\n");
          });
        }
      );
  };
})
+48 −0
Original line number Diff line number Diff line
{ lib
, runCommandLocal
, gfal2-util
}:

# `url` and `urls` should only be overriden via `<pkg>.override`, but not `<pkg>.overrideAttrs`.
{ name ? ""
, pname ? ""
, version ? ""
, urls ? [ ]
, url ? if urls == [ ] then abort "Expect either non-empty `urls` or `url`" else lib.head urls
, hash ? lib.fakeHash
, recursive ? false
, intermediateDestUrls ? [ ]
, extraGfalCopyFlags ? [ ]
, allowSubstitutes ? true
}:

(runCommandLocal name { } ''
  for u in "''${urls[@]}"; do
    gfal-copy "''${gfalCopyFlags[@]}" "$u" "''${intermediateDestUrls[@]}" "$out"
    ret="$?"
    (( ret )) && break
  done
  if (( ret )); then
    echo "gfal-copy failed trying to download from any of the urls" >&2
    exit "$ret"
  fi
'').overrideAttrs (finalAttrs: previousAttrs:
{
  __structuredAttrs = true;
  inherit allowSubstitutes;
  nativeBuildInputs = [ gfal2-util ];
  outputHashAlgo = null;
  outputHashMode = if finalAttrs.recursive then "recursive" else "flat";
  outputHash = hash;
  inherit url;
  urls = if urls == [ ] then lib.singleton url else urls;
  gfalCopyFlags = extraGfalCopyFlags
  ++ lib.optional finalAttrs.recursive "--recursive"
  ;
  inherit recursive intermediateDestUrls;
} // (if (pname != "" && version != "") then {
  inherit pname version;
  name = "${finalAttrs.pname}-${finalAttrs.version}";
} else {
  name = if (name != "") then name else (baseNameOf url);
}))
+19 −0
Original line number Diff line number Diff line
--- a/gfal-SCRIPT_NAME	2022-11-17 00:00:00.000000000 +0000
+++ b/gfal-SCRIPT_NAME	2022-11-17 00:00:00.000000000 +0000
@@ -1,4 +1,4 @@
-#!/bin/sh
+#!/usr/bin/env python3
 # -*- coding: utf-8 -*-
 #
 # Copyright (c) 2022 CERN
@@ -15,10 +15,2 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
-
-# Execute script content with first python interpreter found:
-# * GFAL_PYTHONBIN environment variable
-# * python on the PATH if import gfal2, gfal2_util succeeds
-# * python3 on the PATH if import gfal2, gfal2_util succeeds
-# * python2 on the PATH if import gfal2, gfal2_util succeeds
-# * /usr/bin/python
-"exec" "$(  check_interpreter() { unalias $1 2> /dev/null; unset $1; GFAL_PYTHONBIN=$(command -v $1); [ $GFAL_PYTHONBIN ] && $GFAL_PYTHONBIN -c 'import gfal2, gfal2_util' > /dev/null 2>&1 && { echo $GFAL_PYTHONBIN; unset GFAL_PYTHONBIN; }; }; [ $GFAL_PYTHONBIN ] && echo $GFAL_PYTHONBIN || check_interpreter python || check_interpreter python3 || check_interpreter python2 || echo /usr/bin/python )" "-u" "-Wignore" "$0" "$@"
+6 −0
Original line number Diff line number Diff line
@@ -1834,6 +1834,12 @@ with pkgs;
  github-copilot-cli = callPackage ../tools/misc/github-copilot-cli { };
  # This is to workaround gfal2-python broken against Python 3.12 or later.
  # TODO: Remove these lines after solving the breakage.
  gfal2-util = callPackage ../by-name/gf/gfal2-util/package.nix (lib.optionalAttrs python3Packages.gfal2-python.meta.broken {
    python3Packages = python311Packages;
  });
  gfshare = callPackage ../tools/security/gfshare { };
  gh-actions-cache = callPackage ../tools/misc/gh-actions-cache { };
Loading