Unverified Commit fa83add1 authored by Martin Weinelt's avatar Martin Weinelt Committed by GitHub
Browse files

Merge pull request #271597 from adisbladis/python-runtime-build-time-sep 

python3.pkgs.buildPythonPackage: Separate runtime & build time dependencies
parents d34b0c94 07a221b4

doc/languages-frameworks/python.section.md

+45 −37
Original line number Diff line number Diff line
@@ -116,11 +116,11 @@ buildPythonPackage rec { 
    rm testing/test_argcomplete.py

  '';



  nativeBuildInputs = [

  build-system = [

    setuptools-scm

  ];



  propagatedBuildInputs = [

  dependencies = [

    attrs

    py

    setuptools
@@ -172,7 +172,7 @@ following are specific to `buildPythonPackage`: 
  variable in wrapped programs.

* `pyproject`: Whether the pyproject format should be used. When set to `true`,

  `pypaBuildHook` will be used, and you can add the required build dependencies

  from `build-system.requires` to `nativeBuildInputs`. Note that the pyproject

  from `build-system.requires` to `build-system`. Note that the pyproject

  format falls back to using `setuptools`, so you can use `pyproject = true`

  even if the package only has a `setup.py`. When set to `false`, you can

  use the existing [hooks](#setup-hooks0 or provide your own logic to build the
@@ -206,17 +206,22 @@ build inputs (see "Specifying dependencies"). The following are of special 
interest for Python packages, either because these are primarily used, or

because their behaviour is different:



* `nativeBuildInputs ? []`: Build-time only dependencies. Typically executables

  as well as the items listed in `setup_requires`.

* `nativeBuildInputs ? []`: Build-time only dependencies. Typically executables.

* `build-system ? []`: Build-time only Python dependencies. Items listed in `build-system.requires`/`setup_requires`.

* `buildInputs ? []`: Build and/or run-time dependencies that need to be

  compiled for the host machine. Typically non-Python libraries which are being

  linked.

* `nativeCheckInputs ? []`: Dependencies needed for running the [`checkPhase`](#ssec-check-phase). These

  are added to [`nativeBuildInputs`](#var-stdenv-nativeBuildInputs) when [`doCheck = true`](#var-stdenv-doCheck). Items listed in

  `tests_require` go here.

* `propagatedBuildInputs ? []`: Aside from propagating dependencies,

* `dependencies ? []`: Aside from propagating dependencies,

  `buildPythonPackage` also injects code into and wraps executables with the

  paths included in this list. Items listed in `install_requires` go here.

* `optional-dependencies ? { }`: Optional feature flagged dependencies.  Items listed in `extras_requires` go here.



Aside from propagating dependencies,

  `buildPythonPackage` also injects code into and wraps executables with the

  paths included in this list. Items listed in `extras_requires` go here.



##### Overriding Python packages {#overriding-python-packages}
@@ -299,11 +304,12 @@ python3Packages.buildPythonApplication rec { 
    hash  = "sha256-Pe229rT0aHwA98s+nTHQMEFKZPo/yw6sot8MivFDvAw=";

  };



  nativeBuildInputs = with python3Packages; [

  build-system = with python3Packages; [

    setuptools

    wheel

  ];



  propagatedBuildInputs = with python3Packages; [

  dependencies = with python3Packages; [

    tornado

    python-daemon

  ];
@@ -462,11 +468,11 @@ are used in [`buildPythonPackage`](#buildpythonpackage-function). 
- `eggBuildHook` to skip building for eggs.

- `eggInstallHook` to install eggs.

- `pipBuildHook` to build a wheel using `pip` and PEP 517. Note a build system

  (e.g. `setuptools` or `flit`) should still be added as `nativeBuildInput`.

  (e.g. `setuptools` or `flit`) should still be added as `build-system`.

- `pypaBuildHook` to build a wheel using

  [`pypa/build`](https://pypa-build.readthedocs.io/en/latest/index.html) and

  PEP 517/518. Note a build system (e.g. `setuptools` or `flit`) should still

  be added as `nativeBuildInput`.

  be added as `build-system`.

- `pipInstallHook` to install wheels.

- `pytestCheckHook` to run tests with `pytest`. See [example usage](#using-pytestcheckhook).

- `pythonCatchConflictsHook` to check whether a Python package is not already existing.
@@ -881,7 +887,7 @@ buildPythonPackage rec { 
    hash = "sha256-CP3V73yWSArRHBLUct4hrNMjWZlvaaUlkpm1QP66RWA=";

  };



  nativeBuildInputs = [

  build-system = [

    setuptools

    wheel

  ];
@@ -941,7 +947,7 @@ with import <nixpkgs> {}; 
        hash = "sha256-CP3V73yWSArRHBLUct4hrNMjWZlvaaUlkpm1QP66RWA=";

      };



      nativeBuildInputs = [

      build-system = [

        python311.pkgs.setuptools

        python311.pkgs.wheel

      ];
@@ -977,13 +983,15 @@ that we introduced with the `let` expression. 


#### Handling dependencies {#handling-dependencies}



Our example, `toolz`, does not have any dependencies on other Python packages or

system libraries. According to the manual, [`buildPythonPackage`](#buildpythonpackage-function) uses the

arguments [`buildInputs`](#var-stdenv-buildInputs) and [`propagatedBuildInputs`](#var-stdenv-propagatedBuildInputs) to specify dependencies. If

something is exclusively a build-time dependency, then the dependency should be

included in [`buildInputs`](#var-stdenv-buildInputs), but if it is (also) a runtime dependency, then it

should be added to [`propagatedBuildInputs`](#var-stdenv-propagatedBuildInputs). Test dependencies are considered

build-time dependencies and passed to [`nativeCheckInputs`](#var-stdenv-nativeCheckInputs).

Our example, `toolz`, does not have any dependencies on other Python packages or system libraries.

[`buildPythonPackage`](#buildpythonpackage-function) uses the the following arguments in the following circumstances:



- `dependencies` - For Python runtime dependencies.

- `build-system` - For Python build-time requirements.

- [`buildInputs`](#var-stdenv-buildInputs) - For non-Python build-time requirements.

- [`nativeCheckInputs`](#var-stdenv-nativeCheckInputs) - For test dependencies



Dependencies can belong to multiple arguments, for example if something is both a build time requirement & a runtime dependency.



The following example shows which arguments are given to [`buildPythonPackage`](#buildpythonpackage-function) in

order to build [`datashape`](https://github.com/blaze/datashape).
@@ -1013,12 +1021,12 @@ buildPythonPackage rec { 
    hash = "sha256-FLLvdm1MllKrgTGC6Gb0k0deZeVYvtCCLji/B7uhong=";

  };



  nativeBuildInputs = [

  build-system = [

    setuptools

    wheel

  ];



  propagatedBuildInputs = [

  dependencies = [

    multipledispatch

    numpy

    python-dateutil
@@ -1041,7 +1049,7 @@ buildPythonPackage rec { 
We can see several runtime dependencies, `numpy`, `multipledispatch`, and

`python-dateutil`. Furthermore, we have [`nativeCheckInputs`](#var-stdenv-nativeCheckInputs) with `pytest`.

`pytest` is a test runner and is only used during the [`checkPhase`](#ssec-check-phase) and is

therefore not added to [`propagatedBuildInputs`](#var-stdenv-propagatedBuildInputs).

therefore not added to `dependencies`.



In the previous case we had only dependencies on other Python packages to consider.

Occasionally you have also system libraries to consider. E.g., `lxml` provides
@@ -1068,7 +1076,7 @@ buildPythonPackage rec { 
    hash = "sha256-s9NiusRxFydHzaNRMjjxFcvWxfi45jGb9ql6eJJyQJk=";

  };



  nativeBuildInputs = [

  build-system = [

    setuptools

    wheel

  ];
@@ -1125,7 +1133,7 @@ buildPythonPackage rec { 
    hash = "sha256-9ru2r6kwhUCaskiFoaPNuJCfCVoUL01J40byvRt4kHQ=";

  };



  nativeBuildInputs = [

  build-system = [

    setuptools

    wheel

  ];
@@ -1136,7 +1144,7 @@ buildPythonPackage rec { 
    fftwLongDouble

  ];



  propagatedBuildInputs = [

  dependencies = [

    numpy

    scipy

  ];
@@ -1459,9 +1467,7 @@ mode is activated. 


In the following example, we create a simple environment that has a Python 3.11

version of our package in it, as well as its dependencies and other packages we

like to have in the environment, all specified with [`propagatedBuildInputs`](#var-stdenv-propagatedBuildInputs).

Indeed, we can just add any package we like to have in our environment to

[`propagatedBuildInputs`](#var-stdenv-propagatedBuildInputs).

like to have in the environment, all specified with `dependencies`.



```nix

with import <nixpkgs> {};
@@ -1470,9 +1476,11 @@ with python311Packages; 
buildPythonPackage rec {

  name = "mypackage";

  src = ./path/to/package/source;

  propagatedBuildInputs = [

  dependencies = [

    pytest

    numpy

  ];

  propagatedBuildInputs = [

    pkgs.libsndfile

  ];

}
@@ -1519,7 +1527,7 @@ buildPythonPackage rec { 
    hash = "sha256-CP3V73yWSArRHBLUct4hrNMjWZlvaaUlkpm1QP66RWA=";

  };



  nativeBuildInputs = [

  build-system = [

    setuptools

    wheel

  ];
@@ -1903,8 +1911,8 @@ configure alternatives](#sec-overlays-alternatives-blas-lapack)". 


In a `setup.py` or `setup.cfg` it is common to declare dependencies:



* `setup_requires` corresponds to [`nativeBuildInputs`](#var-stdenv-nativeBuildInputs)

* `install_requires` corresponds to [`propagatedBuildInputs`](#var-stdenv-propagatedBuildInputs)

* `setup_requires` corresponds to `build-system`

* `install_requires` corresponds to `dependencies`

* `tests_require` corresponds to [`nativeCheckInputs`](#var-stdenv-nativeCheckInputs)



### How to enable interpreter optimizations? {#optimizations}
@@ -1928,12 +1936,10 @@ in mypython 


Some packages define optional dependencies for additional features. With

`setuptools` this is called `extras_require` and `flit` calls it

`extras-require`, while PEP 621 calls these `optional-dependencies`. A

method for supporting this is by declaring the extras of a package in its

`passthru`, e.g. in case of the package `dask`

`extras-require`, while PEP 621 calls these `optional-dependencies`.



```nix

passthru.optional-dependencies = {

optional-dependencies = {

  complete = [ distributed ];

};

```
@@ -1941,11 +1947,13 @@ passthru.optional-dependencies = { 
and letting the package requiring the extra add the list to its dependencies



```nix

propagatedBuildInputs = [

dependencies = [

  ...

] ++ dask.optional-dependencies.complete;

```



This method is using `passthru`, meaning that changing `optional-dependencies` of a package won't cause it to rebuild.



Note this method is preferred over adding parameters to builders, as that can

result in packages depending on different variants and thereby causing

collisions.

pkgs/development/interpreters/python/mk-python-derivation.nix

+34 −14
Original line number Diff line number Diff line
@@ -45,6 +45,14 @@ 
# C can import package A propagated by B

, propagatedBuildInputs ? []



# Python module dependencies.

# These are named after PEP-621.

, dependencies ? []

, optional-dependencies ? {}



# Python PEP-517 build systems.

, build-system ? []



# DEPRECATED: use propagatedBuildInputs

, pythonPath ? []
@@ -97,8 +105,6 @@ 


, meta ? {}



, passthru ? {}



, doCheck ? config.doCheckByDefault or false



, disabledTestPaths ? []
@@ -193,10 +199,28 @@ let 
    "setuptools" "wheel"

  ];



  passthru =

    attrs.passthru or { }

    // {

      updateScript = let

        filename = builtins.head (lib.splitString ":" self.meta.position);

      in attrs.passthru.updateScript or [ update-python-libraries filename ];

    }

    // lib.optionalAttrs (dependencies != []) {

      inherit dependencies;

    }

    // lib.optionalAttrs (optional-dependencies != {}) {

      inherit optional-dependencies;

    }

    // lib.optionalAttrs (build-system != []) {

      inherit build-system;

    };



  # Keep extra attributes from `attrs`, e.g., `patchPhase', etc.

  self = toPythonModule (stdenv.mkDerivation ((builtins.removeAttrs attrs [

    "disabled" "checkPhase" "checkInputs" "nativeCheckInputs" "doCheck" "doInstallCheck" "dontWrapPythonPrograms" "catchConflicts" "pyproject" "format"

    "disabledTestPaths" "outputs" "stdenv"

    "dependencies" "optional-dependencies" "build-system"

  ]) // {



    name = namePrefix + name_;
@@ -256,11 +280,11 @@ let 
      pythonNamespacesHook

    ] ++ lib.optionals withDistOutput [

      pythonOutputDistHook

    ] ++ nativeBuildInputs;

    ] ++ nativeBuildInputs ++ build-system;



    buildInputs = validatePythonMatches "buildInputs" (buildInputs ++ pythonPath);



    propagatedBuildInputs = validatePythonMatches "propagatedBuildInputs" (propagatedBuildInputs ++ [

    propagatedBuildInputs = validatePythonMatches "propagatedBuildInputs" (propagatedBuildInputs ++ dependencies ++ [

      # we propagate python even for packages transformed with 'toPythonApplication'

      # this pollutes the PATH but avoids rebuilds

      # see https://github.com/NixOS/nixpkgs/issues/170887 for more context
@@ -292,6 +316,8 @@ let 


    outputs = outputs ++ lib.optional withDistOutput "dist";



    inherit passthru;



    meta = {

      # default to python's platforms

      platforms = python.meta.platforms;
@@ -305,13 +331,7 @@ let 
      disabledTestPaths = lib.escapeShellArgs disabledTestPaths;

  }));



  passthru.updateScript = let

      filename = builtins.head (lib.splitString ":" self.meta.position);

    in attrs.passthru.updateScript or [ update-python-libraries filename ];

in

  if disabled then

    throw "${name} not supported for interpreter ${python.executable}"

else

  self.overrideAttrs (attrs: {

    passthru = lib.recursiveUpdate passthru attrs.passthru;

  })
 
in lib.extendDerivation

  (disabled -> throw "${name} not supported for interpreter ${python.executable}")

  passthru

  self

pkgs/development/python-modules/pendulum/default.nix

+2 −2
Original line number Diff line number Diff line
@@ -20,8 +20,8 @@ buildPythonPackage rec { 
    export HOME=$TMPDIR

  '';



  nativeBuildInputs = [ poetry-core ];

  propagatedBuildInputs = [ python-dateutil pytzdata ]

  build-system = [ poetry-core ];

  dependencies = [ python-dateutil pytzdata ]

  ++ lib.optional (pythonOlder "3.5") typing

  ++ lib.optionals (pythonOlder "3.8") [ importlib-metadata ];

pkgs/development/python-modules/requests/default.nix

+3 −3
Original line number Diff line number Diff line
@@ -30,7 +30,7 @@ buildPythonPackage rec { 
    hash = "sha256-lCxadY+Y15Dq7Ropy27vx/+w0c968Fw9J5Flbb1q0eE=";

  };



  propagatedBuildInputs = [

  dependencies = [

    brotlicffi

    certifi

    charset-normalizer
@@ -38,7 +38,7 @@ buildPythonPackage rec { 
    urllib3

  ];



  passthru.optional-dependencies = {

  optional-dependencies = {

    security = [];

    socks = [

      pysocks
@@ -53,7 +53,7 @@ buildPythonPackage rec { 
    pytest-xdist

    pytestCheckHook

  ]

  ++ passthru.optional-dependencies.socks;

  ++ optional-dependencies.socks;



  disabledTests = [

    # Disable tests that require network access and use httpbin