Merge pull request #215336 from Hjdskes/foundationdb

  "foundationdb61": [

    "setuptools"

  ],

  "foundationdb71": [

    "setuptools"

  ],

  "fountains": [

    "setuptools"

  ],

# This builder is for FoundationDB CMake build system.

{ lib, fetchFromGitHub

, cmake, ninja, boost, python3, openjdk, mono, libressl

, cmake, ninja, python3, openjdk, mono, pkg-config

, msgpack, toml11

, gccStdenv, llvmPackages

, useClang ? false

let

  stdenv = if useClang then llvmPackages.libcxxStdenv else gccStdenv;

  tests = with builtins;

    builtins.replaceStrings [ "\n" ] [ " " ] (lib.fileContents ./test-list.txt);

  tests = builtins.replaceStrings [ "\n" ] [ " " ] (lib.fileContents ./test-list.txt);

  # Only even numbered versions compile on aarch64; odd numbered versions have avx enabled.

  avxEnabled = version:

    let

      isOdd = n: lib.trivial.mod n 2 != 0;

      patch = lib.toInt (lib.versions.patch version);

    in isOdd patch;

  makeFdb =

    { version

    , branch # unused

    , sha256

    , rev ? "refs/tags/${version}"

    , officialRelease ? true

    , patches ? []

    , boost

    , ssl

    }: stdenv.mkDerivation {

        pname = "foundationdb";

        inherit version;

          inherit rev sha256;

        };

        buildInputs = [ libressl boost ];

        nativeBuildInputs = [ cmake ninja python3 openjdk mono ]

        buildInputs = [ ssl boost ]

          ++ lib.optionals (lib.versionAtLeast version "7.1.0") [ msgpack toml11 ];

        nativeBuildInputs = [ pkg-config cmake ninja python3 openjdk mono ]

          ++ lib.optionals useClang [ llvmPackages.lld ];

        separateDebugInfo = true;

        dontFixCmake = true;

        cmakeFlags =

          [ "-DCMAKE_BUILD_TYPE=Release"

            (lib.optionalString officialRelease "-DFDB_RELEASE=TRUE")

          [ (lib.optionalString officialRelease "-DFDB_RELEASE=TRUE")

            # FIXME: why can't libressl be found automatically?

            "-DLIBRESSL_USE_STATIC_LIBS=FALSE"

            "-DLIBRESSL_INCLUDE_DIR=${libressl.dev}"

            "-DLIBRESSL_CRYPTO_LIBRARY=${libressl.out}/lib/libcrypto.so"

            "-DLIBRESSL_SSL_LIBRARY=${libressl.out}/lib/libssl.so"

            "-DLIBRESSL_TLS_LIBRARY=${libressl.out}/lib/libtls.so"

            # Disable CMake warnings for project developers.

            "-Wno-dev"

            # CMake Error at fdbserver/CMakeLists.txt:332 (find_library):

            # >   Could not find lz4_STATIC_LIBRARIES using the following names: liblz4.a

            "-DSSD_ROCKSDB_EXPERIMENTAL=FALSE"

            # FoundationDB's CMake is hardcoded to pull in jemalloc as an external

            # project at build time.

            "-DUSE_JEMALLOC=FALSE"

            # LTO brings up overall build time, but results in much smaller

            # binaries for all users and the cache.

            # Same with LLD when Clang is available.

            (lib.optionalString useClang    "-DUSE_LD=LLD")

            (lib.optionalString (!useClang) "-DUSE_LD=GOLD")

          ] ++ lib.optionals (lib.versionOlder version "7.0.0")

          [ # FIXME: why can't libressl be found automatically?

            "-DLIBRESSL_USE_STATIC_LIBS=FALSE"

            "-DLIBRESSL_INCLUDE_DIR=${ssl.dev}"

            "-DLIBRESSL_CRYPTO_LIBRARY=${ssl.out}/lib/libcrypto.so"

            "-DLIBRESSL_SSL_LIBRARY=${ssl.out}/lib/libssl.so"

            "-DLIBRESSL_TLS_LIBRARY=${ssl.out}/lib/libtls.so"

          ] ++ lib.optionals (lib.versionAtLeast version "7.1.0" && lib.versionOlder version "7.2.0")

          [ # FIXME: why can't openssl be found automatically?

            "-DOPENSSL_USE_STATIC_LIBS=FALSE"

            "-DOPENSSL_CRYPTO_LIBRARY=${ssl.out}/lib/libcrypto.so"

            "-DOPENSSL_SSL_LIBRARY=${ssl.out}/lib/libssl.so"

          ];

        env.NIX_CFLAGS_COMPILE = toString [

          # Needed with GCC 12

          "-Wno-error=missing-template-keyword"

          # Needed to compile on aarch64

          (lib.optionalString stdenv.isAarch64 "-march=armv8-a+crc")

        ];

        inherit patches;

        # fix up the use of the very weird and custom 'fdb_install' command by just

        # replacing it with cmake's ordinary version.

        postPatch = ''

          for x in bindings/c/CMakeLists.txt fdbserver/CMakeLists.txt fdbmonitor/CMakeLists.txt fdbbackup/CMakeLists.txt fdbcli/CMakeLists.txt; do

            substituteInPlace $x --replace 'fdb_install' 'install'

          done

        '';

        # the install phase for cmake is pretty wonky right now since it's not designed to

        # coherently install packages as most linux distros expect -- it's designed to build

        # packaged artifacts that are shipped in RPMs, etc. we need to add some extra code to

        # cmake upstream to fix this, and if we do, i think most of this can go away.

        postInstall = ''

          mv $out/sbin/fdbserver $out/bin/fdbserver

          rm -rf \

            $out/lib/systemd $out/Library $out/usr $out/sbin \

            $out/var $out/log $out/etc

        postInstall = lib.optionalString (lib.versionOlder version "7.0.0") ''

          mv $out/fdbmonitor/fdbmonitor $out/bin/fdbmonitor && rm -rf $out/fdbmonitor

          rm -rf $out/lib/foundationdb/

          mkdir $out/libexec && ln -sfv $out/bin/fdbbackup $out/libexec/backup_agent

          rm -rf $out/Library

          rm -rf $out/lib/foundationdb/

          mkdir $out/include/foundationdb && \

            mv $out/include/*.h $out/include/*.options $out/include/foundationdb

        '' + lib.optionalString (lib.versionAtLeast version "7.0.0") ''

          mv $out/sbin/fdbmonitor $out/bin/fdbmonitor

          mkdir $out/libexec && mv $out/usr/lib/foundationdb/backup_agent/backup_agent $out/libexec/backup_agent

        '' + ''

          mv $out/sbin/fdbserver $out/bin/fdbserver

          rm -rf $out/etc $out/lib/foundationdb $out/lib/systemd $out/log $out/sbin $out/usr $out/var

          # move results into multi outputs

          mkdir -p $dev $lib

          # java bindings

          mkdir -p $lib/share/java

          mv lib/fdb-java-*.jar $lib/share/java/fdb-java.jar

          # include the tests

          mkdir -p $out/share/test

          (cd ../tests && for x in ${tests}; do

            cp --parents $x $out/share/test

          done)

        '';

        outputs = [ "out" "dev" "lib" "pythonsrc" ];

          description = "Open source, distributed, transactional key-value store";

          homepage    = "https://www.foundationdb.org";

          license     = licenses.asl20;

          platforms   = [ "x86_64-linux" ];

          platforms   = [ "x86_64-linux" ]

            ++ lib.optionals (lib.versionAtLeast version "7.1.0" && !(avxEnabled version)) [ "aarch64-linux" ];

          maintainers = with maintainers; [ thoughtpolice lostnet ];

       };

    };

, lib, fetchurl, fetchpatch, fetchFromGitHub

, cmake, ninja, which, findutils, m4, gawk

, python2, python3, openjdk, mono, libressl, boost168

, python2, python3, openjdk, mono, libressl, openssl, boost168, boost178

, pkg-config, msgpack, toml11

}@args:

let

  vsmakeBuild = import ./vsmake.nix args;

  cmakeBuild = import ./cmake.nix (args // {

    gccStdenv    = gccStdenv;

    llvmPackages = llvmPackages;

    boost        = boost168;

  });

  cmakeBuild = import ./cmake.nix args;

  python3-six-patch = fetchpatch {

    name   = "update-python-six.patch";

  foundationdb61 = cmakeBuild {

    version = "6.1.13";

    branch  = "release-6.1";

    sha256  = "10vd694dcnh2pp91mri1m80kfbwjanhiy50c53c5ncqfa6pwvk00";

    boost   = boost168;

    ssl     = libressl;

    patches = [

      ./patches/clang-libcxx.patch

    ];

  };

  foundationdb71 = cmakeBuild {

    version = "7.1.26";

    sha256  = "sha256-IVUFC2Z/nJAeKr/TtEiHAo+1HUeZuSZ2birwJtiYZx0=";

    boost   = boost178;

    ssl     = openssl;

    patches = [

      ./patches/disable-flowbench.patch

      ./patches/don-t-run-tests-requiring-doctest.patch

      ./patches/don-t-use-static-boost-libs.patch

      ./patches/fix-open-with-O_CREAT.patch

    ];

  };

}

diff --git a/CMakeLists.txt b/CMakeLists.txt

index 185b721eb..6752ff32d 100644

--- a/CMakeLists.txt

+++ b/CMakeLists.txt

@@ -185,7 +185,6 @@ endif()

 add_subdirectory(fdbbackup)

 add_subdirectory(contrib)

 add_subdirectory(tests)

-add_subdirectory(flowbench EXCLUDE_FROM_ALL)

 if(WITH_PYTHON AND WITH_C_BINDING)

   add_subdirectory(bindings)

 endif()

-- 

2.38.1

From 10c502fd36df24f1fdbdeff446982ff5247ba20e Mon Sep 17 00:00:00 2001

From: Jente Hidskes Ankarberg <jente@griffin.sh>

Date: Thu, 9 Feb 2023 12:40:21 +0100

Subject: [PATCH] Don't run tests requiring doctest

Doctest is unconditionally pulled in as an external project, which we can't do

---

 bindings/c/CMakeLists.txt | 59 ---------------------------------------

 1 file changed, 59 deletions(-)

diff --git bindings/c/CMakeLists.txt bindings/c/CMakeLists.txt

index b1a187b99..25b626819 100644

--- a/bindings/c/CMakeLists.txt

+++ b/bindings/c/CMakeLists.txt

@@ -84,7 +84,6 @@ if(NOT WIN32)

     test/mako/mako.h

     test/mako/utils.c

     test/mako/utils.h)

-  add_subdirectory(test/unit/third_party)

   find_package(Threads REQUIRED)

   set(UNIT_TEST_SRCS

     test/unit/unit_tests.cpp

@@ -93,10 +92,6 @@ if(NOT WIN32)

   set(UNIT_TEST_VERSION_510_SRCS test/unit/unit_tests_version_510.cpp)

   set(TRACE_PARTIAL_FILE_SUFFIX_TEST_SRCS test/unit/trace_partial_file_suffix_test.cpp)

-  set(DISCONNECTED_TIMEOUT_UNIT_TEST_SRCS

-    test/unit/disconnected_timeout_tests.cpp

-    test/unit/fdb_api.cpp

-    test/unit/fdb_api.hpp)

   set(API_TESTER_SRCS

     test/apitester/fdb_c_api_tester.cpp

@@ -128,11 +123,7 @@ if(NOT WIN32)

     add_library(fdb_c_txn_size_test OBJECT test/txn_size_test.c test/test.h)

     add_library(fdb_c_client_memory_test OBJECT test/client_memory_test.cpp test/unit/fdb_api.cpp test/unit/fdb_api.hpp)

     add_library(mako OBJECT ${MAKO_SRCS})

-    add_library(fdb_c_setup_tests OBJECT test/unit/setup_tests.cpp)

-    add_library(fdb_c_unit_tests OBJECT ${UNIT_TEST_SRCS})

-    add_library(fdb_c_unit_tests_version_510 OBJECT ${UNIT_TEST_VERSION_510_SRCS})

     add_library(trace_partial_file_suffix_test OBJECT ${TRACE_PARTIAL_FILE_SUFFIX_TEST_SRCS})

-    add_library(disconnected_timeout_unit_tests OBJECT ${DISCONNECTED_TIMEOUT_UNIT_TEST_SRCS})

     add_library(fdb_c_api_tester OBJECT ${API_TESTER_SRCS})

   else()

     add_executable(fdb_c_performance_test test/performance_test.c test/test.h)

@@ -140,11 +131,7 @@ if(NOT WIN32)

     add_executable(fdb_c_txn_size_test test/txn_size_test.c test/test.h)

     add_executable(fdb_c_client_memory_test test/client_memory_test.cpp test/unit/fdb_api.cpp test/unit/fdb_api.hpp)

     add_executable(mako ${MAKO_SRCS})

-    add_executable(fdb_c_setup_tests test/unit/setup_tests.cpp)

-    add_executable(fdb_c_unit_tests ${UNIT_TEST_SRCS})

-    add_executable(fdb_c_unit_tests_version_510 ${UNIT_TEST_VERSION_510_SRCS})

     add_executable(trace_partial_file_suffix_test ${TRACE_PARTIAL_FILE_SUFFIX_TEST_SRCS})

-    add_executable(disconnected_timeout_unit_tests ${DISCONNECTED_TIMEOUT_UNIT_TEST_SRCS})

     add_executable(fdb_c_api_tester ${API_TESTER_SRCS})

     strip_debug_symbols(fdb_c_performance_test)

     strip_debug_symbols(fdb_c_ryw_benchmark)

@@ -155,20 +142,7 @@ if(NOT WIN32)

   target_link_libraries(fdb_c_ryw_benchmark PRIVATE fdb_c Threads::Threads)

   target_link_libraries(fdb_c_txn_size_test PRIVATE fdb_c Threads::Threads)

   target_link_libraries(fdb_c_client_memory_test PRIVATE fdb_c Threads::Threads)

-

-  add_dependencies(fdb_c_setup_tests doctest)

-  add_dependencies(fdb_c_unit_tests doctest)

-  add_dependencies(fdb_c_unit_tests_version_510 doctest)

-  add_dependencies(disconnected_timeout_unit_tests doctest)

-  target_include_directories(fdb_c_setup_tests PUBLIC ${DOCTEST_INCLUDE_DIR})

-  target_include_directories(fdb_c_unit_tests PUBLIC ${DOCTEST_INCLUDE_DIR})

-  target_include_directories(fdb_c_unit_tests_version_510 PUBLIC ${DOCTEST_INCLUDE_DIR})

-  target_include_directories(disconnected_timeout_unit_tests PUBLIC ${DOCTEST_INCLUDE_DIR})

-  target_link_libraries(fdb_c_setup_tests PRIVATE fdb_c Threads::Threads)

-  target_link_libraries(fdb_c_unit_tests PRIVATE fdb_c Threads::Threads fdbclient)

-  target_link_libraries(fdb_c_unit_tests_version_510 PRIVATE fdb_c Threads::Threads)

   target_link_libraries(trace_partial_file_suffix_test PRIVATE fdb_c Threads::Threads flow)

-  target_link_libraries(disconnected_timeout_unit_tests PRIVATE fdb_c Threads::Threads)

 if(USE_SANITIZER)

   target_link_libraries(fdb_c_api_tester PRIVATE fdb_c toml11_target Threads::Threads fmt::fmt boost_asan)

@@ -203,46 +177,13 @@ endif()

     DEPENDS fdb_c

     COMMENT "Copy libfdb_c to use as external client for test")

   add_custom_target(external_client DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/libfdb_c_external.so)

-  add_dependencies(fdb_c_unit_tests external_client)

-  add_dependencies(disconnected_timeout_unit_tests external_client)

   add_dependencies(fdb_c_api_tester external_client)

-  add_fdbclient_test(

-    NAME fdb_c_setup_tests

-    COMMAND $<TARGET_FILE:fdb_c_setup_tests>)

-  add_fdbclient_test(

-    NAME fdb_c_unit_tests

-    COMMAND $<TARGET_FILE:fdb_c_unit_tests>

-            @CLUSTER_FILE@

-            fdb)

-  add_fdbclient_test(

-    NAME fdb_c_unit_tests_version_510

-    COMMAND $<TARGET_FILE:fdb_c_unit_tests_version_510>

-            @CLUSTER_FILE@

-            fdb)

   add_fdbclient_test(

     NAME trace_partial_file_suffix_test

     COMMAND $<TARGET_FILE:trace_partial_file_suffix_test>

             @CLUSTER_FILE@

             fdb)

-  add_fdbclient_test(

-    NAME fdb_c_external_client_unit_tests

-    COMMAND $<TARGET_FILE:fdb_c_unit_tests>

-            @CLUSTER_FILE@

-            fdb

-            ${CMAKE_CURRENT_BINARY_DIR}/libfdb_c_external.so

-            )

-  add_unavailable_fdbclient_test(

-    NAME disconnected_timeout_unit_tests

-    COMMAND $<TARGET_FILE:disconnected_timeout_unit_tests>

-            @CLUSTER_FILE@

-            )

-  add_unavailable_fdbclient_test(

-    NAME disconnected_timeout_external_client_unit_tests

-    COMMAND $<TARGET_FILE:disconnected_timeout_unit_tests>

-            @CLUSTER_FILE@

-            ${CMAKE_CURRENT_BINARY_DIR}/libfdb_c_external.so

-            )

   add_fdbclient_test(

     NAME fdb_c_api_tests

     DISABLE_LOG_DUMP

-- 

2.37.1 (Apple Git-137.1)