Commit a689220a authored by Steve Purcell's avatar Steve Purcell
Browse files

duckdb: Incorporate changes from #257452

parent f40761c7

pkgs/development/libraries/duckdb/default.nix

+23 −43
Original line number Diff line number Diff line
@@ -2,6 +2,7 @@ 
, stdenv

, fetchFromGitHub

, cmake

, git

, ninja

, openssl

, openjdk11
@@ -20,35 +21,24 @@ stdenv.mkDerivation rec { 
  src = fetchFromGitHub {

    owner = pname;

    repo = pname;

    deepClone = true;

    rev = "v${version}";

    hash = "sha256-EKvDH7RwOC4Gu/lturrfnGpzXnJ9azIwAFeuVoa6L/Y=";

  };



  patches = [ ./version.patch ];

  nativeBuildInputs = [ cmake git ninja ];



  postPatch = ''

    substituteInPlace CMakeLists.txt --subst-var-by DUCKDB_VERSION "v${version}"

  '';



  nativeBuildInputs = [ cmake ninja ];

  buildInputs = [ openssl ]

    ++ lib.optionals withJdbc [ openjdk11 ]

    ++ lib.optionals withOdbc [ unixODBC ];



  cmakeFlags = [

    "-DBUILD_AUTOCOMPLETE_EXTENSION=ON"

    # use similar flags to what is defined in ${src}/.github/workflow/{LinuxRelease,OSX}.yml

    "-DBUILD_ICU_EXTENSION=ON"

    "-DDEBUG_STACKTRACE=1"

    "-DBUILD_PARQUET_EXTENSION=ON"

    "-DDUCKDB_EXTENSION_CONFIGS=${src}/.github/config/bundled_extensions.cmake"

    "-DBUILD_TPCH_EXTENSION=ON"

    "-DBUILD_TPCDS_EXTENSION=ON"

    "-DBUILD_FTS_EXTENSION=ON"

    "-DBUILD_HTTPFS_EXTENSION=ON"

    "-DBUILD_VISUALIZER_EXTENSION=ON"

    "-DBUILD_JSON_EXTENSION=ON"

    "-DBUILD_JEMALLOC_EXTENSION=ON"

    "-DBUILD_EXCEL_EXTENSION=ON"

    "-DBUILD_INET_EXTENSION=ON"

    "-DBUILD_TPCE=ON"

    "-DBUILD_ODBC_DRIVER=${enableFeature withOdbc}"

    "-DJDBC_DRIVER=${enableFeature withJdbc}"

  ] ++ lib.optionals doInstallCheck [
@@ -66,44 +56,34 @@ stdenv.mkDerivation rec { 


  installCheckPhase =

    let

      excludes = map (pattern: "exclude:'${pattern}'") [

        "[s3]"

      excludes = map (pattern: "exclude:'${pattern}'") ([

        "Test using a remote optimizer pass in case thats important to someone"

        "Test closing database during long running query"

        "test/common/test_cast_hugeint.test"

        "test/sql/copy/csv/test_csv_remote.test"

        "test/sql/copy/parquet/test_parquet_remote.test"

        "test/sql/copy/parquet/test_parquet_remote_foreign_files.test"

        "test/sql/storage/compression/chimp/chimp_read.test"

        "test/sql/storage/compression/chimp/chimp_read_float.test"

        "test/sql/storage/compression/patas/patas_compression_ratio.test_coverage"

        "test/sql/storage/compression/patas/patas_read.test"

        "test/sql/json/read_json_objects.test"

        "test/sql/json/read_json.test"

        "test/sql/copy/parquet/parquet_5968.test"

        "test/fuzzer/pedro/buffer_manager_out_of_memory.test"

        "test/sql/storage/compression/bitpacking/bitpacking_size_calculation.test"

        "test/sql/copy/csv/parallel/csv_parallel_httpfs.test"

        "test/sql/copy/csv/parallel/test_parallel_csv.test"

        "test/sql/copy/csv/test_csv_httpfs_prepared.test"

        "test/sql/copy/csv/test_csv_httpfs.test"

        "test/sql/copy/csv/test_csv_remote.test"

        "test/sql/copy/parquet/delta_byte_array_length_mismatch.test"

        "test/sql/function/timestamp/test_icu_strptime.test"

        "test/sql/timezone/test_icu_timezone.test"

        "test/sql/copy/parquet/delta_byte_array_multiple_pages.test"

        "test/sql/copy/parquet/parquet_5968.test"

        "test/sql/copy/parquet/snowflake_lineitem.test"

        "test/sql/copy/parquet/test_parquet_force_download.test"

        "test/sql/copy/parquet/delta_byte_array_multiple_pages.test"

        "test/sql/copy/csv/test_csv_httpfs_prepared.test"

        "test/sql/copy/csv/test_csv_httpfs.test"

        "test/sql/copy/csv/parallel/test_parallel_csv.test"

        "test/sql/copy/csv/parallel/csv_parallel_httpfs.test"

        "test/common/test_cast_struct.test"

        # test is order sensitive

        "test/sql/copy/parquet/parquet_glob.test"

        "test/sql/copy/parquet/test_parquet_remote_foreign_files.test"

        "test/sql/copy/parquet/test_parquet_remote.test"

        "test/sql/json/table/read_json_objects.test"

        "test/sql/json/table/read_json.test"

        "test/sql/settings/test_disabled_file_system_httpfs.test"

        "test/sql/storage/compression/bitpacking/bitpacking_size_calculation.test"



        # these are only hidden if no filters are passed in

        "[!hide]"

        # this test apparently never terminates

        "test/sql/copy/csv/auto/test_csv_auto.test"

      ] ++ lib.optionals stdenv.isAarch64 [

        "test/sql/aggregate/aggregates/test_kurtosis.test"

        "test/sql/aggregate/aggregates/test_skewness.test"

        "test/sql/function/list/aggregates/skewness.test"

      ];

      ]);

    in

    ''

      runHook preInstallCheck

pkgs/development/libraries/duckdb/version.patch

deleted100644 → 0
+0 −58
Original line number Diff line number Diff line 
diff --git a/CMakeLists.txt b/CMakeLists.txt

index 349af6acf7..7ffec0b4cb 100644

--- a/CMakeLists.txt

+++ b/CMakeLists.txt

@@ -196,52 +196,7 @@ if(${CMAKE_SYSTEM_NAME} STREQUAL "SunOS")

   set(SUN TRUE)

 endif()

 

-find_package(Git)

-if(Git_FOUND)

-  if (NOT DEFINED GIT_COMMIT_HASH)

-    execute_process(

-            COMMAND ${GIT_EXECUTABLE} log -1 --format=%h

-            WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}

-            RESULT_VARIABLE GIT_RESULT

-            OUTPUT_VARIABLE GIT_COMMIT_HASH

-            OUTPUT_STRIP_TRAILING_WHITESPACE)

-  endif()

-  execute_process(

-          COMMAND ${GIT_EXECUTABLE} describe --tags --abbrev=0

-          WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}

-          OUTPUT_VARIABLE GIT_LAST_TAG

-          OUTPUT_STRIP_TRAILING_WHITESPACE)

-  execute_process(

-          COMMAND ${GIT_EXECUTABLE} describe --tags --long

-          WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}

-          OUTPUT_VARIABLE GIT_ITERATION

-          OUTPUT_STRIP_TRAILING_WHITESPACE)

-else()

-  message("Git NOT FOUND")

-endif()

-

-if(GIT_RESULT EQUAL "0")

-  string(REGEX REPLACE "v([0-9]+).[0-9]+.[0-9]+" "\\1" DUCKDB_MAJOR_VERSION "${GIT_LAST_TAG}")

-  string(REGEX REPLACE "v[0-9]+.([0-9]+).[0-9]+" "\\1" DUCKDB_MINOR_VERSION "${GIT_LAST_TAG}")

-  string(REGEX REPLACE "v[0-9]+.[0-9]+.([0-9]+)" "\\1" DUCKDB_PATCH_VERSION "${GIT_LAST_TAG}")

-  string(REGEX REPLACE ".*-([0-9]+)-.*" "\\1" DUCKDB_DEV_ITERATION "${GIT_ITERATION}")

-

-  if(DUCKDB_DEV_ITERATION EQUAL 0)

-    # on a tag; directly use the version

-    set(DUCKDB_VERSION "${GIT_LAST_TAG}")

-  else()

-    # not on a tag, increment the patch version by one and add a -devX suffix

-    math(EXPR DUCKDB_PATCH_VERSION "${DUCKDB_PATCH_VERSION}+1")

-    set(DUCKDB_VERSION "v${DUCKDB_MAJOR_VERSION}.${DUCKDB_MINOR_VERSION}.${DUCKDB_PATCH_VERSION}-dev${DUCKDB_DEV_ITERATION}")

-  endif()

-else()

-  # fallback for when building from tarball

-  set(DUCKDB_MAJOR_VERSION 0)

-  set(DUCKDB_MINOR_VERSION 0)

-  set(DUCKDB_PATCH_VERSION 1)

-  set(DUCKDB_DEV_ITERATION 0)

-  set(DUCKDB_VERSION "v${DUCKDB_MAJOR_VERSION}.${DUCKDB_MINOR_VERSION}.${DUCKDB_PATCH_VERSION}-dev${DUCKDB_DEV_ITERATION}")

-endif()

+set(DUCKDB_VERSION "@DUCKDB_VERSION@")

 

 message(STATUS "git hash ${GIT_COMMIT_HASH}, version ${DUCKDB_VERSION}")

pkgs/development/python-modules/duckdb-engine/default.nix

+7 −2
Original line number Diff line number Diff line
@@ -14,7 +14,7 @@ 


buildPythonPackage rec {

  pname = "duckdb-engine";

  version = "0.7.3";

  version = "0.9.2";

  format = "pyproject";



  disabled = pythonOlder "3.7";
@@ -23,9 +23,11 @@ buildPythonPackage rec { 
    repo = "duckdb_engine";

    owner = "Mause";

    rev = "refs/tags/v${version}";

    hash = "sha256-Z9m1+Bc/csWKdPDuwf82xX0qOiD1Y5LBgJjUlLntAO8=";

    hash = "sha256-T02nGF+YlughRQPinb0I3NC6xsarh4+qRhG8YfhTvhI=";

  };



   patches = [ ./remote_data.patch ];



  nativeBuildInputs = [

    poetry-core

  ];
@@ -42,8 +44,11 @@ buildPythonPackage rec { 
  disabledTests = [

    # this test tries to download the httpfs extension

    "test_preload_extension"

    "test_motherduck"

    # test should be skipped based on sqlalchemy version but isn't and fails

    "test_commit"

    # rowcount no longer generates an attribute error.

    "test_rowcount"

  ];



  nativeCheckInputs = [

pkgs/development/python-modules/duckdb-engine/remote_data.patch

0 → 100644
+24 −0
Original line number Diff line number Diff line 
diff --git a/duckdb_engine/tests/test_basic.py b/duckdb_engine/tests/test_basic.py

index 302636f..ed20f12 100644

--- a/duckdb_engine/tests/test_basic.py

+++ b/duckdb_engine/tests/test_basic.py

@@ -183,7 +183,6 @@ def test_get_views(engine: Engine) -> None:

 

 

 @mark.skipif(os.uname().machine == "aarch64", reason="not supported on aarch64")

-@mark.remote_data

 def test_preload_extension() -> None:

     duckdb.default_connection.execute("INSTALL httpfs")

     engine = create_engine(

diff --git a/duckdb_engine/tests/test_integration.py b/duckdb_engine/tests/test_integration.py

index 349c976..bf3dbaa 100644

--- a/duckdb_engine/tests/test_integration.py

+++ b/duckdb_engine/tests/test_integration.py

@@ -24,7 +24,6 @@ def test_integration(engine: Engine) -> None:

         conn.execute(text("select * from test_df"))

 

 

-@mark.remote_data

 @mark.skipif(

     "dev" in duckdb.__version__, reason="md extension not available for dev builds"  # type: ignore[attr-defined]

 )
 
 No newline at end of file

pkgs/development/python-modules/duckdb/default.nix

+13 −6
Original line number Diff line number Diff line
@@ -2,8 +2,10 @@ 
, buildPythonPackage

, duckdb

, fsspec

, git

, google-cloud-storage

, numpy

, openssl

, pandas

, psutil

, pybind11
@@ -12,9 +14,12 @@ 
}:



buildPythonPackage rec {

  inherit (duckdb) pname version src patches;

  inherit (duckdb) pname version src;

  format = "setuptools";



  BUILD_HTTPFS = 1;

  patches = [ ./setup.patch ];



  postPatch = ''

    # we can't use sourceRoot otherwise patches don't apply, because the patches apply to the C++ library

    cd tools/pythonpkg
@@ -22,8 +27,7 @@ buildPythonPackage rec { 
    # 1. let nix control build cores

    # 2. unconstrain setuptools_scm version

    substituteInPlace setup.py \

      --replace "multiprocessing.cpu_count()" "$NIX_BUILD_CORES" \

      --replace "setuptools_scm<7.0.0" "setuptools_scm"

      --replace "multiprocessing.cpu_count()" "$NIX_BUILD_CORES"



    # avoid dependency on mypy

    rm tests/stubs/test_stubs.py
@@ -32,20 +36,23 @@ buildPythonPackage rec { 
  SETUPTOOLS_SCM_PRETEND_VERSION = version;



  nativeBuildInputs = [

    git

    pybind11

    setuptools-scm

  ];



  buildInputs = [ openssl ];



  propagatedBuildInputs = [

    numpy

    pandas

  ];



  nativeCheckInputs = [

    fsspec

    google-cloud-storage

    psutil

    pytestCheckHook

    fsspec

  ];



  disabledTests = [