Merge pull request #200100 from ShamrockLee/gfal2

{ python3Packages }:

with python3Packages; toPythonApplication gfal2-util

{ lib

, stdenv

, callPackage

, fetchFromGitHub

  # Native build inputs

, cmake

, pkg-config

  # General build inputs

, glib

, gtest

, json_c

, openldap

  # Plugin build inputs

, cryptopp

, davix-copy

, dcap

, libssh2

, libuuid

, pugixml

, xrootd

  # For enablePluginStatus.https only

, gsoap

}:

stdenv.mkDerivation (finalAttrs: {

  pname = "gfal2";

  version = "2.22.2";

  src = fetchFromGitHub {

    owner = "cern-fts";

    repo = "gfal2";

    rev = "v${finalAttrs.version}";

    hash = "sha256-xcM29mZRUrnSE0//rHMaJFgPBeT6E4WdB9tCFa/y5+g=";

  };

  passthru.enablePluginStatus = {

    # TODO: Change back to `true` once dcap is fixed on Darwin.

    dcap = !dcap.meta.broken;

    file = true;

    gridftp = false;

    # davix-copy's dependency gsoap is currently only available on Linux.

    # TODO: Change back to `true` once gsoap is fixed on Darwin.

    http = lib.meta.availableOn stdenv.hostPlatform gsoap;

    lfc = false;

    # Break because of redundant `-luuid`. This needs to be fixed from the gfal2 upstream.

    # TODO: Change back to `true` once fixed.

    mock = !stdenv.isDarwin;

    rfio = false;

    sftp = true;

    srm = false;

    xrootd = true;

  };

  passthru.tests = (

    # Enable only one plugin in each test case,

    # to ensure that they gets their dependency when invoked separately.

    lib.listToAttrs

      (map

        (pluginName: lib.nameValuePair

          "gfal2-${pluginName}"

          (finalAttrs.finalPackage.overrideAttrs (previousAttrs: {

            passthru = previousAttrs.passthru // {

              enablePluginStatus = lib.mapAttrs (n: v: n == pluginName) previousAttrs.passthru.enablePluginStatus;

            };

          })))

        (lib.filter (lib.flip lib.getAttr finalAttrs.passthru.enablePluginStatus) (lib.attrNames finalAttrs.passthru.enablePluginStatus))

      )

  ) // {

    # Disable all plugins in this test case.

    gfal2-minimal = finalAttrs.finalPackage.overrideAttrs (previousAttrs: {

      passthru.enablePluginStatus = lib.mapAttrs (n: v: false) previousAttrs.passthru.enablePluginStatus;

    });

  };

  nativeBuildInputs = [

    cmake

    pkg-config

  ];

  buildInputs = lib.unique (

    [

      glib

      json_c

      # gfal2 version older than 2.21.1 fails to see openldap 2.5+

      # and will complain

      # bin/ld: cannot find -lldap_r: No such file or directory

      # See https://github.com/cern-fts/gfal2/blob/aa24462bb67e259e525f26fb5feb97050a8c5c61/RELEASE-NOTES

      openldap

      pugixml # Optional, for MDS Cache.

    ]

    ++ lib.optionals finalAttrs.passthru.enablePluginStatus.dcap [ dcap ]

    ++ lib.optionals finalAttrs.passthru.enablePluginStatus.http [ cryptopp davix-copy ]

    ++ lib.optionals finalAttrs.passthru.enablePluginStatus.mock [ libuuid ]

    ++ lib.optionals finalAttrs.passthru.enablePluginStatus.sftp [ libssh2 ]

    ++ lib.optionals finalAttrs.passthru.enablePluginStatus.xrootd [ xrootd libuuid ]

  );

  cmakeFlags = (

    map

      (pluginName: "-DPLUGIN_${lib.toUpper pluginName}=${lib.toUpper (lib.boolToString finalAttrs.passthru.enablePluginStatus.${pluginName})}")

      (lib.attrNames finalAttrs.passthru.enablePluginStatus)

  )

  ++ [ "-DSKIP_TESTS=${lib.toUpper (lib.boolToString (!finalAttrs.doCheck))}" ]

  ++ lib.optionals finalAttrs.doCheck [ "-DGTEST_INCLUDE_DIR=${gtest.dev}/include" ]

  ++ lib.optionals finalAttrs.passthru.enablePluginStatus.http [ "-DCRYPTOPP_INCLUDE_DIRS=${cryptopp.dev}/include/cryptopp" ]

  ++ lib.optionals finalAttrs.passthru.enablePluginStatus.xrootd [ "-DXROOTD_INCLUDE_DIR=${xrootd.dev}/include/xrootd" ]

  ;

  doCheck = stdenv.hostPlatform.isLinux;

  checkInputs = [

    gtest

  ];

  meta = with lib; {

    description = "Multi-protocol data management library by CERN";

    longDescription = ''

      GFAL (Grid File Access Library )

      is a C library providing an abstraction layer of

      the grid storage system complexity.

      The complexity of the grid is hidden from the client side

      behind a simple common POSIX API.

    '';

    homepage = "https://github.com/cern-fts/gfal2";

    license = licenses.asl20;

    platforms = platforms.all;

    maintainers = with maintainers; [ ShamrockLee ];

    mainProgram = "gfal2";

  };

})

{ lib

, buildPythonPackage

, fetchFromGitHub

, cmake

, pkg-config

, boost

, gfal2

, glib

, pythonAtLeast

  # For tests

, gfal2-util ? null

}:

buildPythonPackage rec {

  pname = "gfal2-python";

  version = "1.12.2";

  src = fetchFromGitHub {

    owner = "cern-fts";

    repo = "gfal2-python";

    rev = "v${version}";

    hash = "sha256-Xk+gLTrqfWb0kGB6QhnM62zAHVFb8rRAqCIBxn0V824=";

  };

  nativeBuildInputs = [

    cmake

    pkg-config

  ];

  buildInputs = [

    boost

    gfal2

    glib

  ];

  # We don't want setup.py to (re-)execute cmake in buildPhase

  # Besides, this package is totally handled by CMake, which means no additional configuration is needed.

  dontConfigure = true;

  pythonImportsCheck = [

    "gfal2"

  ];

  passthru = {

    inherit gfal2;

    tests = {

      inherit gfal2-util;

    }

    // lib.optionalAttrs (gfal2-util != null) gfal2-util.tests or { };

  };

  meta = with lib; {

    description = "Python binding for gfal2";

    homepage = "https://github.com/cern-fts/gfal2-python";

    license = licenses.asl20;

    maintainers = with maintainers; [ ShamrockLee ];

    # It currently fails to build against Python 3.12 or later,

    # complaining CMake faililng to find Python include path, library path and site package path.

    broken = pythonAtLeast "3.12";

  };

}

{ lib

, buildPythonPackage

, callPackage

, fetchFromGitHub

, runCommandLocal

  # Build inputs

, gfal2-python

  # For tests

, xrootd # pkgs.xrootd

}:

(buildPythonPackage rec {

  pname = "gfal2-util";

  version = "1.8.1";

  src = fetchFromGitHub {

    owner = "cern-fts";

    repo = "gfal2-util";

    rev = "v${version}";

    hash = "sha256-3JbJgKD17aYkrB/aaww7IQU8fLFrTCh868KWlLPxmlk=";

  };

  # Replace the ad-hoc python executable finding

  # and change the shebangs from `#!/bin/sh` to `#!/usr/bin/env python`

  # for fixup phase to work correctly.

  postPatch = ''

    for script in src/gfal-*; do

      patch "$script" ${./gfal-util-script.patch}

    done

  '';

  propagatedBuildInputs = [

    gfal2-python

  ];

  pythonImportsCheck = [

    "gfal2_util"

  ];

  meta = with lib; {

    description = "CLI for gfal2";

    homepage = "https://github.com/cern-fts/gfal2-utils";

    license = licenses.asl20;

    maintainers = with maintainers; [ ShamrockLee ];

  };

}).overrideAttrs (finalAttrs: previousAttrs: lib.recursiveUpdate previousAttrs {

  passthru = {

    inherit (gfal2-python) gfal2;

    fetchGfal2 = lib.makeOverridable (callPackage ./fetchgfal2.nix { gfal2-util = finalAttrs.finalPackage; });

    # With these functionality tests, it should be safe to merge version bumps once all the tests are passed.

    tests =

      let

        # Use the the bin output hash of gfal2-util as version to ensure that

        # the test gets rebuild everytime gfal2-util gets rebuild

        versionFODTests = finalAttrs.version + "-" + lib.substring (lib.stringLength builtins.storeDir + 1) 32 "${self}";

        self = finalAttrs.finalPackage;

      in

      lib.optionalAttrs gfal2-python.gfal2.enablePluginStatus.xrootd (

        let

          # Test against a real-world dataset from CERN Open Data

          # borrowed from `xrootd.tests`.

          urlTestFile = xrootd.tests.test-xrdcp.url;

          hashTestFile = xrootd.tests.test-xrdcp.outputHash;

          urlTestDir = dirOf urlTestFile;

        in

        {

          test-copy-file-xrootd = finalAttrs.passthru.fetchGfal2 {

            url = urlTestFile;

            hash = hashTestFile;

            extraGfalCopyFlags = [ "--verbose" ];

            pname = "gfal2-util-test-copy-file-xrootd";

            version = versionFODTests;

            allowSubstitutes = false;

          };

          test-copy-dir-xrootd = finalAttrs.passthru.fetchGfal2 {

            url = urlTestDir;

            hash = "sha256-vOahIhvx1oE9sfkqANMGUvGeLHS737wyfYWo4rkvrxw=";

            recursive = true;

            extraGfalCopyFlags = [ "--verbose" ];

            pname = "gfal2-util-test-copy-dir-xrootd";

            version = versionFODTests;

            allowSubstitutes = false;

          };

          test-ls-dir-xrootd = (runCommandLocal "test-gfal2-util-ls-dir-xrootd" { } ''

            set -eu -o pipefail

            gfal-ls "$url" | grep "$baseNameExpected" | tee "$out"

          '').overrideAttrs (finalAttrs: previousAttrs: {

            pname = previousAttrs.name;

            version = versionFODTests;

            name = "${finalAttrs.pname}-${finalAttrs.version}";

            nativeBuildInputs = [ self ];

            url = urlTestDir;

            baseNameExpected = baseNameOf urlTestFile;

            outputHashMode = "flat";

            outputHashAlgo = "sha256";

            outputHash = builtins.hashString finalAttrs.outputHashAlgo (finalAttrs.baseNameExpected + "\n");

          });

        }

      );

  };

})

{ lib

, runCommandLocal

, gfal2-util

}:

# `url` and `urls` should only be overriden via `<pkg>.override`, but not `<pkg>.overrideAttrs`.

{ name ? ""

, pname ? ""

, version ? ""

, urls ? [ ]

, url ? if urls == [ ] then abort "Expect either non-empty `urls` or `url`" else lib.head urls

, hash ? lib.fakeHash

, recursive ? false

, intermediateDestUrls ? [ ]

, extraGfalCopyFlags ? [ ]

, allowSubstitutes ? true

}:

(runCommandLocal name { } ''

  for u in "''${urls[@]}"; do

    gfal-copy "''${gfalCopyFlags[@]}" "$u" "''${intermediateDestUrls[@]}" "$out"

    ret="$?"

    (( ret )) && break

  done

  if (( ret )); then

    echo "gfal-copy failed trying to download from any of the urls" >&2

    exit "$ret"

  fi

'').overrideAttrs (finalAttrs: previousAttrs:

{

  __structuredAttrs = true;

  inherit allowSubstitutes;

  nativeBuildInputs = [ gfal2-util ];

  outputHashAlgo = null;

  outputHashMode = if finalAttrs.recursive then "recursive" else "flat";

  outputHash = hash;

  inherit url;

  urls = if urls == [ ] then lib.singleton url else urls;

  gfalCopyFlags = extraGfalCopyFlags

  ++ lib.optional finalAttrs.recursive "--recursive"

  ;

  inherit recursive intermediateDestUrls;

} // (if (pname != "" && version != "") then {

  inherit pname version;

  name = "${finalAttrs.pname}-${finalAttrs.version}";

} else {

  name = if (name != "") then name else (baseNameOf url);

}))