python312Packages.swh-export: init at 1.8.0 (#395541)

{

  lib,

  stdenv,

  fetchFromGitHub,

  fetchurl,

  fetchpatch,

  cmake,

  gtest,

  lz4,

  protobuf,

  snappy,

  zlib,

  zstd,

}:

let

  orc-format = fetchurl {

    name = "orc-format-1.0.0.tar.gz";

    url = "https://www.apache.org/dyn/closer.lua/orc/orc-format-1.0.0/orc-format-1.0.0.tar.gz?action=download";

    hash = "sha256-c5+uX/lLH4ErQTB3KANhBFv5LlEO8Es0phDiOpRdjNU=";

  };

in

stdenv.mkDerivation (finalAttrs: {

  pname = "apache-orc";

  version = "2.1.1";

  src = fetchFromGitHub {

    owner = "apache";

    repo = "orc";

    tag = "v${finalAttrs.version}";

    hash = "sha256-Ib02mIUQaLEVwIBv5xdKDyX+FeI8vhh9+5hM7miKwHo=";

  };

  patches = [

    # Patch that adds 2 missing imports in source files

    # To be removed this PR land: https://github.com/apache/orc/pull/2175

    (fetchpatch {

      url = "https://github.com/apache/orc/commit/fb20db2440226da6b92d38ce260e5b850d2f0092.patch";

      hash = "sha256-rHGECXJoBPgZ62yZciYdSMq4pGnVt75lxkHyO46IiyQ=";

    })

  ];

  nativeBuildInputs = [

    cmake

  ];

  buildInputs = [

    gtest

    lz4

    protobuf

    snappy

    zlib

    zstd

  ];

  cmakeFlags = [

    (lib.cmakeFeature "CMAKE_BUILD_TYPE" "Release")

    (lib.cmakeBool "BUILD_JAVA" false)

    (lib.cmakeBool "STOP_BUILD_ON_WARNING" true)

    (lib.cmakeBool "INSTALL_VENDORED_LIBS" false)

  ];

  env = {

    GTEST_HOME = gtest.dev;

    LZ4_ROOT = lz4;

    ORC_FORMAT_URL = orc-format;

    PROTOBUF_HOME = protobuf;

    SNAPPY_ROOT = snappy.dev;

    ZLIB_ROOT = zlib.dev;

    ZSTD_ROOT = zstd.dev;

  };

  meta = {

    changelog = "https://github.com/apache/orc/releases/tag/v${finalAttrs.version}";

    description = "The smallest, fastest columnar storage for Hadoop workloads";

    homepage = "https://github.com/apache/orc/";

    license = lib.licenses.asl20;

    maintainers = with lib.maintainers; [ drupol ];

    platforms = lib.platforms.all;

  };

})

  writeShellApplication,

  withSwhPythonPackages ? [

    python3Packages.swh-auth

    python3Packages.swh-export

    python3Packages.swh-model

    python3Packages.swh-objstorage

    python3Packages.swh-scanner

{

  lib,

  buildPythonPackage,

  fetchFromGitHub,

  pybind11,

  setuptools,

  pytestCheckHook,

  tzdata,

  python,

  pkgs,

}:

buildPythonPackage rec {

  pname = "pyorc";

  version = "0.10.0";

  pyproject = true;

  src = fetchFromGitHub {

    owner = "noirello";

    repo = "pyorc";

    tag = "v${version}";

    hash = "sha256-2w3Qh6g+Yg+D10kTow9YR6B6FhQ+z2DvgDy5GtYxH4g=";

  };

  build-system = [

    pybind11

    setuptools

  ];

  env = {

    PYORC_SKIP_ORC_BUILD = "true";

  };

  buildInputs = [

    pkgs.lz4

    pkgs.protobuf

    pkgs.snappy

    pkgs.zlib

    pkgs.zstd

    pkgs.apache-orc

  ];

  preCheck = ''

    # provide timezone data, works only on linux

    export TZDIR=${tzdata}/${python.sitePackages}/tzdata/zoneinfo

    substituteInPlace "tests/compare/test_writer_cmp.py" \

      --replace-fail "deps/bin/orc-contents" "orc-contents"

    substituteInPlace "tests/compare/test_reader_cmp.py" \

      --replace-fail "deps/bin/orc-metadata" "orc-metadata"

    mkdir -p deps

    ln -s "${pkgs.apache-orc.src}/examples" "deps/"

  '';

  pythonImportsCheck = [

    "pyorc"

  ];

  nativeCheckInputs = [

    pytestCheckHook

    pkgs.apache-orc

  ];

  meta = {

    changelog = "https://github.com/noirello/pyorc/blob/${version}/CHANGELOG.rst";

    description = "Python module for Apache ORC file format";

    homepage = "https://github.com/noirello/pyorc";

    license = lib.licenses.asl20;

    maintainers = with lib.maintainers; [ drupol ];

  };

}

{

  lib,

  buildPythonPackage,

  fetchFromGitLab,

  setuptools,

  kafka-python-ng,

  port-for,

  pytest,

}:

buildPythonPackage rec {

  pname = "pytest-kafka";

  version = "0.8.1";

  pyproject = true;

  src = fetchFromGitLab {

    owner = "karolinepauls";

    repo = "pytest-kafka";

    tag = "v${version}";

    hash = "sha256-OR8SpNswbPOVtAcFuZgrZJR5K6wPb1TS5leybKWr3zY=";

  };

  build-system = [ setuptools ];

  dependencies = [

    kafka-python-ng

    port-for

    pytest

  ];

  pythonImportsCheck = [ "pytest_kafka" ];

  # Tests depends on a kafka server running

  doCheck = false;

  meta = {

    description = "Pytest fixture factories for Zookeeper, Kafka server and Kafka consumer";

    homepage = "https://gitlab.com/karolinepauls/pytest-kafka";

    license = lib.licenses.mit;

    maintainers = with lib.maintainers; [ drupol ];

  };

}

{

  lib,

  buildPythonPackage,

  fetchFromGitLab,

  setuptools,

  setuptools-scm,

  boto3,

  click,

  tqdm,

  pyorc,

  plyvel,

  python,

  types-requests,

  swh-core,

  swh-journal,

  swh-model,

  swh-storage,

  pytestCheckHook,

  pytest-kafka,

  pytest-mock,

  tzdata,

  pkgs,

}:

buildPythonPackage rec {

  pname = "swh-export";

  version = "1.8.0";

  pyproject = true;

  src = fetchFromGitLab {

    domain = "gitlab.softwareheritage.org";

    group = "swh";

    owner = "devel";

    repo = "swh-export";

    tag = "v${version}";

    hash = "sha256-n97MMYn7EmTrv411YSxUD1+zfbFB8KOSns44N3NqqV8=";

  };

  build-system = [

    setuptools

    setuptools-scm

  ];

  dependencies = [

    boto3

    click

    tqdm

    pyorc

    plyvel

    types-requests

    swh-core

    swh-journal

    swh-model

    swh-storage

  ];

  preCheck = ''

    # provide timezone data, works only on linux

    export TZDIR=${tzdata}/${python.sitePackages}/tzdata/zoneinfo

  '';

  pythonImportsCheck = [ "swh.export" ];

  nativeCheckInputs = [

    pytestCheckHook

    pytest-kafka

    pytest-mock

    pkgs.zstd

    pkgs.pv

  ];

  disabledTests = [

    # I don't know how to fix the following error

    # E       fixture 'kafka_server' not found

    "test_parallel_journal_processor"

    "test_parallel_journal_processor_origin"

    "test_parallel_journal_processor_origin_visit_status"

    "test_parallel_journal_processor_offsets"

    "test_parallel_journal_processor_masked"

    "test_parallel_journal_processor_masked_origin"

    "test_parallel_journal_processor_masked_origin_visit_statuses"

  ];

  meta = {

    description = "Software Heritage dataset tools";

    homepage = "https://gitlab.softwareheritage.org/swh/devel/swh-export";

    license = lib.licenses.gpl3Only;

    maintainers = with lib.maintainers; [ drupol ];

  };

}