Loading maintainers/maintainer-list.nix +6 −0 Original line number Diff line number Diff line Loading @@ -4788,6 +4788,12 @@ githubId = 16470252; name = "Gemini Lasswell"; }; gbpdt = { email = "nix@pdtpartners.com"; github = "gbpdt"; githubId = 25106405; name = "Graham Bennett"; }; gbtb = { email = "goodbetterthebeast3@gmail.com"; github = "gbtb"; Loading pkgs/development/python-modules/apache-airflow/default.nix +78 −28 Original line number Diff line number Diff line Loading @@ -11,15 +11,19 @@ , cattrs , clickclick , colorlog , connexion , cron-descriptor , croniter , cryptography , dataclasses , deprecated , dill , flask , flask_login , flask-wtf , flask-appbuilder , flask-caching , flask-session , flask-wtf , GitPython , graphviz , gunicorn Loading @@ -32,13 +36,16 @@ , jinja2 , jsonschema , lazy-object-proxy , linkify-it-py , lockfile , markdown , markupsafe , marshmallow-oneofschema , mdit-py-plugins , numpy , openapi-spec-validator , pandas , pathspec , pendulum , psutil , pygments Loading @@ -58,20 +65,27 @@ , tabulate , tenacity , termcolor , typing-extensions , unicodecsv , werkzeug , pytestCheckHook , freezegun , mkYarnPackage , writeScript # Extra airflow providers to enable , enabledProviders ? [] }: let version = "2.3.3"; version = "2.3.4"; airflow-src = fetchFromGitHub rec { owner = "apache"; repo = "airflow"; rev = "refs/tags/${version}"; sha256 = "sha256-N+6ljfSo6+UvSAnvDav6G0S49JZ1VJwxmaiKPV3/DjA="; # Required because the GitHub archive tarballs don't appear to include tests leaveDotGit = true; sha256 = "sha256-rxvLyz/hvZ6U8QKy9MiVofU0qeeo7OHctAj2PkxLh2c="; }; # airflow bundles a web interface, which is built using webpack by an undocumented shell script in airflow's source tree. Loading @@ -87,6 +101,12 @@ let distPhase = "true"; # The webpack license plugin tries to create /licenses when given the # original relative path postPatch = '' sed -i 's!../../../../licenses/LICENSES-ui.txt!licenses/LICENSES-ui.txt!' webpack.config.js ''; configurePhase = '' cp -r $node_modules node_modules ''; Loading @@ -102,6 +122,13 @@ let ''; }; # Import generated file with metadata for provider dependencies and imports. # Enable additional providers using enabledProviders above. providers = import ./providers.nix; getProviderDeps = provider: map (dep: python.pkgs.${dep}) providers.${provider}.deps; getProviderImports = provider: providers.${provider}.imports; providerDependencies = lib.concatMap getProviderDeps enabledProviders; providerImports = lib.concatMap getProviderImports enabledProviders; in buildPythonPackage rec { pname = "apache-airflow"; Loading @@ -119,14 +146,18 @@ buildPythonPackage rec { cattrs clickclick colorlog connexion cron-descriptor croniter cryptography deprecated dill flask flask-appbuilder flask-caching flask_login flask-session flask-wtf flask_login GitPython graphviz gunicorn Loading @@ -138,13 +169,16 @@ buildPythonPackage rec { jinja2 jsonschema lazy-object-proxy linkify-it-py lockfile markdown markupsafe marshmallow-oneofschema mdit-py-plugins numpy openapi-spec-validator pandas pathspec pendulum psutil pygments Loading @@ -163,13 +197,14 @@ buildPythonPackage rec { tabulate tenacity termcolor typing-extensions unicodecsv werkzeug ] ++ lib.optionals (pythonOlder "3.7") [ dataclasses ] ++ lib.optionals (pythonOlder "3.9") [ importlib-metadata ]; ] ++ providerDependencies; buildInputs = [ airflow-frontend Loading @@ -180,29 +215,15 @@ buildPythonPackage rec { pytestCheckHook ]; # By default, source code of providers is included but unusable due to missing # transitive dependencies. To enable a provider, add it to extraProviders # above INSTALL_PROVIDERS_FROM_SOURCES = "true"; postPatch = '' substituteInPlace setup.cfg \ --replace "attrs>=20.0, <21.0" "attrs" \ --replace "cattrs~=1.1, <1.7.0" "cattrs" \ --replace "colorlog>=4.0.2, <6.0" "colorlog" \ --replace "croniter>=0.3.17, <1.1" "croniter" \ --replace "docutils<0.17" "docutils" \ --replace "flask-login>=0.3, <0.5" "flask-login" \ --replace "flask-wtf>=0.14.3, <0.15" "flask-wtf" \ --replace "flask>=1.1.0, <2.0" "flask" \ --replace "importlib_resources~=1.4" "importlib_resources" \ --replace "itsdangerous>=1.1.0, <2.0" "itsdangerous" \ --replace "markupsafe>=1.1.1, <2.0" "markupsafe" \ --replace "pyjwt<2" "pyjwt" \ --replace "python-slugify>=3.0.0,<5.0" "python-slugify" \ --replace "sqlalchemy_jsonfield~=1.0" "sqlalchemy-jsonfield" \ --replace "tenacity~=6.2.0" "tenacity" \ --replace "werkzeug~=1.0, >=1.0.1" "werkzeug" substituteInPlace tests/core/test_core.py \ --replace "/bin/bash" "${stdenv.shell}" --replace "colorlog>=4.0.2, <5.0" "colorlog" \ --replace "flask-login>=0.6.2" "flask-login" '' + lib.optionalString stdenv.isDarwin '' # Fix failing test on Hydra substituteInPlace airflow/utils/db.py \ Loading @@ -214,7 +235,11 @@ buildPythonPackage rec { "--prefix PYTHONPATH : $PYTHONPATH" ]; preCheck = '' pythonImportsCheck = [ "airflow" ] ++ providerImports; checkPhase = '' export HOME=$(mktemp -d) export AIRFLOW_HOME=$HOME export AIRFLOW__CORE__UNIT_TEST_MODE=True Loading @@ -238,12 +263,37 @@ buildPythonPackage rec { cp -rv ${airflow-frontend}/static/dist $out/lib/${python.libPrefix}/site-packages/airflow/www/static ''; # Updates yarn.lock and package.json passthru.updateScript = writeScript "update.sh" '' #!/usr/bin/env nix-shell #!nix-shell -i bash -p common-updater-scripts curl pcre "python3.withPackages (ps: with ps; [ pyyaml ])" yarn2nix set -euo pipefail # Get new version new_version="$(curl -s https://airflow.apache.org/docs/apache-airflow/stable/release_notes.html | pcregrep -o1 'Airflow ([0-9.]+).' | head -1)" update-source-version ${pname} "$new_version" # Update frontend cd ./pkgs/development/python-modules/apache-airflow curl -O https://raw.githubusercontent.com/apache/airflow/$new_version/airflow/www/yarn.lock curl -O https://raw.githubusercontent.com/apache/airflow/$new_version/airflow/www/package.json # Note: for 2.3.4 a manual change was needed to get a fully resolved URL for # caniuse-lite@1.0.30001312 (with the sha after the #). The error from yarn # was 'Can't make a request in offline mode' from yarn. Corrected install by # manually running yarn add caniuse-lite@1.0.30001312 and copying the # requisite section from the generated yarn.lock. yarn2nix > yarn.nix # update provider dependencies ./update-providers.py ''; meta = with lib; { description = "Programmatically author, schedule and monitor data pipelines"; homepage = "https://airflow.apache.org/"; license = licenses.asl20; maintainers = with maintainers; [ bhipple costrouc ingenieroariel ]; # requires extremely outdated versions of multiple dependencies broken = true; maintainers = with maintainers; [ bhipple gbpdt ingenieroariel ]; }; } pkgs/development/python-modules/apache-airflow/package.json +60 −31 Original line number Diff line number Diff line { "name": "airflow-frontend", "version": "2.1.1rc1", "name": "airflow-www", "version": "1.0.0", "description": "Apache Airflow is a platform to programmatically author, schedule and monitor workflows.", "scripts": { "test": "echo \"Error: no test specified\" && exit 1", "dev": "NODE_ENV=dev webpack --watch --colors --progress --debug --output-pathinfo --devtool eval-cheap-source-map --mode development", "prod": "NODE_ENV=production node --max_old_space_size=4096 ./node_modules/webpack/bin/webpack.js -p --colors --progress", "build": "NODE_ENV=production webpack --colors --progress", "lint": "eslint --ignore-path=.eslintignore --ext .js,.html .", "lint:fix": "eslint --fix --ignore-path=.eslintignore --ext .js,.html ." "test": "jest", "dev": "NODE_ENV=development webpack --watch --progress --devtool eval-cheap-source-map --mode development", "prod": "NODE_ENV=production node --max_old_space_size=4096 ./node_modules/webpack/bin/webpack.js --mode production --progress", "build": "NODE_ENV=production webpack --progress --mode production", "lint": "eslint --ignore-path=.eslintignore --ext .js,.jsx,.ts,.tsx . && tsc --noEmit", "lint:fix": "eslint --fix --ignore-path=.eslintignore --ext .js,.jsx,.ts,.tsx . && tsc --noEmit" }, "author": "Apache", "license": "Apache-2.0", Loading @@ -28,53 +28,82 @@ "flask" ], "devDependencies": { "babel": "^6.23.0", "babel-core": "^6.26.3", "babel-eslint": "^10.1.0", "@babel/core": "^7.18.5", "@babel/eslint-parser": "^7.18.2", "@babel/plugin-transform-runtime": "^7.16.0", "@babel/preset-env": "^7.16.0", "@babel/preset-react": "^7.16.0", "@babel/preset-typescript": "^7.17.12", "@testing-library/jest-dom": "^5.16.0", "@testing-library/react": "^13.0.0", "@types/react": "^18.0.12", "@types/react-dom": "^18.0.5", "@typescript-eslint/eslint-plugin": "^5.13.0", "@typescript-eslint/parser": "^5.0.0", "babel-jest": "^27.3.1", "babel-loader": "^8.1.0", "babel-plugin-css-modules-transform": "^1.6.1", "babel-polyfill": "^6.26.0", "clean-webpack-plugin": "^3.0.0", "copy-webpack-plugin": "^6.0.3", "css-loader": "^3.4.2", "eslint": "^7.5.0", "eslint-config-airbnb-base": "^14.2.0", "css-loader": "5.2.7", "css-minimizer-webpack-plugin": "^4.0.0", "eslint": "^8.6.0", "eslint-config-airbnb": "^19.0.4", "eslint-config-airbnb-typescript": "^17.0.0", "eslint-plugin-html": "^6.0.2", "eslint-plugin-import": "^2.22.0", "eslint-plugin-import": "^2.25.3", "eslint-plugin-jsx-a11y": "^6.5.0", "eslint-plugin-node": "^11.1.0", "eslint-plugin-promise": "^4.2.1", "eslint-plugin-react": "^7.30.0", "eslint-plugin-react-hooks": "^4.5.0", "eslint-plugin-standard": "^4.0.1", "file-loader": "^6.0.0", "imports-loader": "^1.1.0", "mini-css-extract-plugin": "1.6.0", "jest": "^27.3.1", "mini-css-extract-plugin": "^1.6.2", "moment": "^2.29.3", "moment-locales-webpack-plugin": "^1.2.0", "optimize-css-assets-webpack-plugin": "6.0.0", "nock": "^13.2.4", "style-loader": "^1.2.1", "stylelint": "^13.6.1", "stylelint-config-standard": "^20.0.0", "terser-webpack-plugin": "<5.0.0", "typescript": "^4.6.3", "url-loader": "4.1.0", "webpack": "^4.16.3", "webpack-cli": "^3.1.0", "webpack-manifest-plugin": "^2.2.0" "webpack": "^5.73.0", "webpack-cli": "^4.0.0", "webpack-license-plugin": "^4.2.1", "webpack-manifest-plugin": "^4.0.0" }, "dependencies": { "@chakra-ui/react": "^2.2.0", "@emotion/cache": "^11.9.3", "@emotion/react": "^11.9.3", "@emotion/styled": "^11", "axios": "^0.26.0", "bootstrap-3-typeahead": "^4.0.2", "camelcase-keys": "^7.0.0", "codemirror": "^5.59.1", "d3": "^3.4.4", "d3-shape": "^2.1.0", "d3-tip": "^0.9.1", "dagre-d3": "^0.6.4", "datatables.net": "^1.10.23", "datatables.net-bs": "^1.10.23", "datatables.net": "^1.11.4", "datatables.net-bs": "^1.11.4", "eonasdan-bootstrap-datetimepicker": "^4.17.47", "jquery": ">=3.4.0", "jshint": "^2.12.0", "moment-timezone": "^0.5.28", "framer-motion": "^6.0.0", "jquery": ">=3.5.0", "jshint": "^2.13.4", "lodash": "^4.17.21", "moment-timezone": "^0.5.34", "nvd3": "^1.8.6", "redoc": "^2.0.0-rc.48", "react": "^18.0.0", "react-dom": "^18.0.0", "react-icons": "^4.3.1", "react-query": "^3.39.1", "react-router-dom": "^6.3.0", "react-table": "^7.8.0", "redoc": "^2.0.0-rc.72", "url-search-params-polyfill": "^8.1.0" }, "resolutions": { "lodash": "^4.17.21" } } pkgs/development/python-modules/apache-airflow/providers.nix 0 → 100644 +303 −0 File added.Preview size limit exceeded, changes collapsed. Show changes pkgs/development/python-modules/apache-airflow/update-providers.py 0 → 100755 +226 −0 Original line number Diff line number Diff line #! /usr/bin/env python3 from itertools import chain import json import logging from pathlib import Path import os import re import subprocess import sys from typing import Dict, List, Optional, Set, TextIO from urllib.request import urlopen from urllib.error import HTTPError import yaml PKG_SET = "pkgs.python3Packages" # If some requirements are matched by multiple or no Python packages, the # following can be used to choose the correct one PKG_PREFERENCES = { "dnspython": "dnspython", "google-api-python-client": "google-api-python-client", "psycopg2-binary": "psycopg2", "requests_toolbelt": "requests-toolbelt", } # Requirements missing from the airflow provider metadata EXTRA_REQS = { "sftp": ["pysftp"], } def get_version(): with open(os.path.dirname(sys.argv[0]) + "/default.nix") as fh: # A version consists of digits, dots, and possibly a "b" (for beta) m = re.search('version = "([\\d\\.b]+)";', fh.read()) return m.group(1) def get_file_from_github(version: str, path: str): with urlopen( f"https://raw.githubusercontent.com/apache/airflow/{version}/{path}" ) as response: return yaml.safe_load(response) def repository_root() -> Path: return Path(os.path.dirname(sys.argv[0])) / "../../../.." def dump_packages() -> Dict[str, Dict[str, str]]: # Store a JSON dump of Nixpkgs' python3Packages output = subprocess.check_output( [ "nix-env", "-f", repository_root(), "-qa", "-A", PKG_SET, "--arg", "config", "{ allowAliases = false; }", "--json", ] ) return json.loads(output) def remove_version_constraint(req: str) -> str: return re.sub(r"[=><~].*$", "", req) def name_to_attr_path(req: str, packages: Dict[str, Dict[str, str]]) -> Optional[str]: if req in PKG_PREFERENCES: return f"{PKG_SET}.{PKG_PREFERENCES[req]}" attr_paths = [] names = [req] # E.g. python-mpd2 is actually called python3.6-mpd2 # instead of python-3.6-python-mpd2 inside Nixpkgs if req.startswith("python-") or req.startswith("python_"): names.append(req[len("python-") :]) for name in names: # treat "-" and "_" equally name = re.sub("[-_]", "[-_]", name) # python(minor).(major)-(pname)-(version or unstable-date) # we need the version qualifier, or we'll have multiple matches # (e.g. pyserial and pyserial-asyncio when looking for pyserial) pattern = re.compile( f"^python\\d+\\.\\d+-{name}-(?:\\d|unstable-.*)", re.I ) for attr_path, package in packages.items(): # logging.debug("Checking match for %s with %s", name, package["name"]) if pattern.match(package["name"]): attr_paths.append(attr_path) # Let's hope there's only one derivation with a matching name assert len(attr_paths) <= 1, f"{req} matches more than one derivation: {attr_paths}" if attr_paths: return attr_paths[0] return None def provider_reqs_to_attr_paths(reqs: List, packages: Dict) -> List: no_version_reqs = map(remove_version_constraint, reqs) filtered_reqs = [ req for req in no_version_reqs if not re.match(r"^apache-airflow", req) ] attr_paths = [] for req in filtered_reqs: attr_path = name_to_attr_path(req, packages) if attr_path is not None: # Add attribute path without "python3Packages." prefix pname = attr_path[len(PKG_SET + ".") :] attr_paths.append(pname) else: # If we can't find it, we just skip and warn the user logging.warning("Could not find package attr for %s", req) return attr_paths def get_cross_provider_reqs( provider: str, provider_reqs: Dict, cross_provider_deps: Dict, seen: List = None ) -> Set: # Unfortunately there are circular cross-provider dependencies, so keep a # list of ones we've seen already seen = seen or [] reqs = set(provider_reqs[provider]) if len(cross_provider_deps[provider]) > 0: reqs.update( chain.from_iterable( get_cross_provider_reqs( d, provider_reqs, cross_provider_deps, seen + [provider] ) if d not in seen else [] for d in cross_provider_deps[provider] ) ) return reqs def get_provider_reqs(version: str, packages: Dict) -> Dict: provider_dependencies = get_file_from_github( version, "generated/provider_dependencies.json" ) provider_reqs = {} cross_provider_deps = {} for provider, provider_data in provider_dependencies.items(): provider_reqs[provider] = list( provider_reqs_to_attr_paths(provider_data["deps"], packages) ) + EXTRA_REQS.get(provider, []) cross_provider_deps[provider] = [ d for d in provider_data["cross-providers-deps"] if d != "common.sql" ] transitive_provider_reqs = {} # Add transitive cross-provider reqs for provider in provider_reqs: transitive_provider_reqs[provider] = get_cross_provider_reqs( provider, provider_reqs, cross_provider_deps ) return transitive_provider_reqs def get_provider_yaml(version: str, provider: str) -> Dict: provider_dir = provider.replace(".", "/") path = f"airflow/providers/{provider_dir}/provider.yaml" try: return get_file_from_github(version, path) except HTTPError: logging.warning("Couldn't get provider yaml for %s", provider) return {} def get_provider_imports(version: str, providers) -> Dict: provider_imports = {} for provider in providers: provider_yaml = get_provider_yaml(version, provider) imports: List[str] = [] if "hooks" in provider_yaml: imports.extend( chain.from_iterable( hook["python-modules"] for hook in provider_yaml["hooks"] ) ) if "operators" in provider_yaml: imports.extend( chain.from_iterable( operator["python-modules"] for operator in provider_yaml["operators"] ) ) provider_imports[provider] = imports return provider_imports def to_nix_expr(provider_reqs: Dict, provider_imports: Dict, fh: TextIO) -> None: fh.write("# Warning: generated by update-providers.py, do not update manually\n") fh.write("{\n") for provider, reqs in provider_reqs.items(): provider_name = provider.replace(".", "_") fh.write(f" {provider_name} = {{\n") fh.write( " deps = [ " + " ".join(sorted(f'"{req}"' for req in reqs)) + " ];\n" ) fh.write( " imports = [ " + " ".join(sorted(f'"{imp}"' for imp in provider_imports[provider])) + " ];\n" ) fh.write(" };\n") fh.write("}\n") def main() -> None: logging.basicConfig(level=logging.INFO) version = get_version() packages = dump_packages() logging.info("Generating providers.nix for version %s", version) provider_reqs = get_provider_reqs(version, packages) provider_imports = get_provider_imports(version, provider_reqs.keys()) with open("providers.nix", "w") as fh: to_nix_expr(provider_reqs, provider_imports, fh) if __name__ == "__main__": main() Loading
maintainers/maintainer-list.nix +6 −0 Original line number Diff line number Diff line Loading @@ -4788,6 +4788,12 @@ githubId = 16470252; name = "Gemini Lasswell"; }; gbpdt = { email = "nix@pdtpartners.com"; github = "gbpdt"; githubId = 25106405; name = "Graham Bennett"; }; gbtb = { email = "goodbetterthebeast3@gmail.com"; github = "gbtb"; Loading
pkgs/development/python-modules/apache-airflow/default.nix +78 −28 Original line number Diff line number Diff line Loading @@ -11,15 +11,19 @@ , cattrs , clickclick , colorlog , connexion , cron-descriptor , croniter , cryptography , dataclasses , deprecated , dill , flask , flask_login , flask-wtf , flask-appbuilder , flask-caching , flask-session , flask-wtf , GitPython , graphviz , gunicorn Loading @@ -32,13 +36,16 @@ , jinja2 , jsonschema , lazy-object-proxy , linkify-it-py , lockfile , markdown , markupsafe , marshmallow-oneofschema , mdit-py-plugins , numpy , openapi-spec-validator , pandas , pathspec , pendulum , psutil , pygments Loading @@ -58,20 +65,27 @@ , tabulate , tenacity , termcolor , typing-extensions , unicodecsv , werkzeug , pytestCheckHook , freezegun , mkYarnPackage , writeScript # Extra airflow providers to enable , enabledProviders ? [] }: let version = "2.3.3"; version = "2.3.4"; airflow-src = fetchFromGitHub rec { owner = "apache"; repo = "airflow"; rev = "refs/tags/${version}"; sha256 = "sha256-N+6ljfSo6+UvSAnvDav6G0S49JZ1VJwxmaiKPV3/DjA="; # Required because the GitHub archive tarballs don't appear to include tests leaveDotGit = true; sha256 = "sha256-rxvLyz/hvZ6U8QKy9MiVofU0qeeo7OHctAj2PkxLh2c="; }; # airflow bundles a web interface, which is built using webpack by an undocumented shell script in airflow's source tree. Loading @@ -87,6 +101,12 @@ let distPhase = "true"; # The webpack license plugin tries to create /licenses when given the # original relative path postPatch = '' sed -i 's!../../../../licenses/LICENSES-ui.txt!licenses/LICENSES-ui.txt!' webpack.config.js ''; configurePhase = '' cp -r $node_modules node_modules ''; Loading @@ -102,6 +122,13 @@ let ''; }; # Import generated file with metadata for provider dependencies and imports. # Enable additional providers using enabledProviders above. providers = import ./providers.nix; getProviderDeps = provider: map (dep: python.pkgs.${dep}) providers.${provider}.deps; getProviderImports = provider: providers.${provider}.imports; providerDependencies = lib.concatMap getProviderDeps enabledProviders; providerImports = lib.concatMap getProviderImports enabledProviders; in buildPythonPackage rec { pname = "apache-airflow"; Loading @@ -119,14 +146,18 @@ buildPythonPackage rec { cattrs clickclick colorlog connexion cron-descriptor croniter cryptography deprecated dill flask flask-appbuilder flask-caching flask_login flask-session flask-wtf flask_login GitPython graphviz gunicorn Loading @@ -138,13 +169,16 @@ buildPythonPackage rec { jinja2 jsonschema lazy-object-proxy linkify-it-py lockfile markdown markupsafe marshmallow-oneofschema mdit-py-plugins numpy openapi-spec-validator pandas pathspec pendulum psutil pygments Loading @@ -163,13 +197,14 @@ buildPythonPackage rec { tabulate tenacity termcolor typing-extensions unicodecsv werkzeug ] ++ lib.optionals (pythonOlder "3.7") [ dataclasses ] ++ lib.optionals (pythonOlder "3.9") [ importlib-metadata ]; ] ++ providerDependencies; buildInputs = [ airflow-frontend Loading @@ -180,29 +215,15 @@ buildPythonPackage rec { pytestCheckHook ]; # By default, source code of providers is included but unusable due to missing # transitive dependencies. To enable a provider, add it to extraProviders # above INSTALL_PROVIDERS_FROM_SOURCES = "true"; postPatch = '' substituteInPlace setup.cfg \ --replace "attrs>=20.0, <21.0" "attrs" \ --replace "cattrs~=1.1, <1.7.0" "cattrs" \ --replace "colorlog>=4.0.2, <6.0" "colorlog" \ --replace "croniter>=0.3.17, <1.1" "croniter" \ --replace "docutils<0.17" "docutils" \ --replace "flask-login>=0.3, <0.5" "flask-login" \ --replace "flask-wtf>=0.14.3, <0.15" "flask-wtf" \ --replace "flask>=1.1.0, <2.0" "flask" \ --replace "importlib_resources~=1.4" "importlib_resources" \ --replace "itsdangerous>=1.1.0, <2.0" "itsdangerous" \ --replace "markupsafe>=1.1.1, <2.0" "markupsafe" \ --replace "pyjwt<2" "pyjwt" \ --replace "python-slugify>=3.0.0,<5.0" "python-slugify" \ --replace "sqlalchemy_jsonfield~=1.0" "sqlalchemy-jsonfield" \ --replace "tenacity~=6.2.0" "tenacity" \ --replace "werkzeug~=1.0, >=1.0.1" "werkzeug" substituteInPlace tests/core/test_core.py \ --replace "/bin/bash" "${stdenv.shell}" --replace "colorlog>=4.0.2, <5.0" "colorlog" \ --replace "flask-login>=0.6.2" "flask-login" '' + lib.optionalString stdenv.isDarwin '' # Fix failing test on Hydra substituteInPlace airflow/utils/db.py \ Loading @@ -214,7 +235,11 @@ buildPythonPackage rec { "--prefix PYTHONPATH : $PYTHONPATH" ]; preCheck = '' pythonImportsCheck = [ "airflow" ] ++ providerImports; checkPhase = '' export HOME=$(mktemp -d) export AIRFLOW_HOME=$HOME export AIRFLOW__CORE__UNIT_TEST_MODE=True Loading @@ -238,12 +263,37 @@ buildPythonPackage rec { cp -rv ${airflow-frontend}/static/dist $out/lib/${python.libPrefix}/site-packages/airflow/www/static ''; # Updates yarn.lock and package.json passthru.updateScript = writeScript "update.sh" '' #!/usr/bin/env nix-shell #!nix-shell -i bash -p common-updater-scripts curl pcre "python3.withPackages (ps: with ps; [ pyyaml ])" yarn2nix set -euo pipefail # Get new version new_version="$(curl -s https://airflow.apache.org/docs/apache-airflow/stable/release_notes.html | pcregrep -o1 'Airflow ([0-9.]+).' | head -1)" update-source-version ${pname} "$new_version" # Update frontend cd ./pkgs/development/python-modules/apache-airflow curl -O https://raw.githubusercontent.com/apache/airflow/$new_version/airflow/www/yarn.lock curl -O https://raw.githubusercontent.com/apache/airflow/$new_version/airflow/www/package.json # Note: for 2.3.4 a manual change was needed to get a fully resolved URL for # caniuse-lite@1.0.30001312 (with the sha after the #). The error from yarn # was 'Can't make a request in offline mode' from yarn. Corrected install by # manually running yarn add caniuse-lite@1.0.30001312 and copying the # requisite section from the generated yarn.lock. yarn2nix > yarn.nix # update provider dependencies ./update-providers.py ''; meta = with lib; { description = "Programmatically author, schedule and monitor data pipelines"; homepage = "https://airflow.apache.org/"; license = licenses.asl20; maintainers = with maintainers; [ bhipple costrouc ingenieroariel ]; # requires extremely outdated versions of multiple dependencies broken = true; maintainers = with maintainers; [ bhipple gbpdt ingenieroariel ]; }; }
pkgs/development/python-modules/apache-airflow/package.json +60 −31 Original line number Diff line number Diff line { "name": "airflow-frontend", "version": "2.1.1rc1", "name": "airflow-www", "version": "1.0.0", "description": "Apache Airflow is a platform to programmatically author, schedule and monitor workflows.", "scripts": { "test": "echo \"Error: no test specified\" && exit 1", "dev": "NODE_ENV=dev webpack --watch --colors --progress --debug --output-pathinfo --devtool eval-cheap-source-map --mode development", "prod": "NODE_ENV=production node --max_old_space_size=4096 ./node_modules/webpack/bin/webpack.js -p --colors --progress", "build": "NODE_ENV=production webpack --colors --progress", "lint": "eslint --ignore-path=.eslintignore --ext .js,.html .", "lint:fix": "eslint --fix --ignore-path=.eslintignore --ext .js,.html ." "test": "jest", "dev": "NODE_ENV=development webpack --watch --progress --devtool eval-cheap-source-map --mode development", "prod": "NODE_ENV=production node --max_old_space_size=4096 ./node_modules/webpack/bin/webpack.js --mode production --progress", "build": "NODE_ENV=production webpack --progress --mode production", "lint": "eslint --ignore-path=.eslintignore --ext .js,.jsx,.ts,.tsx . && tsc --noEmit", "lint:fix": "eslint --fix --ignore-path=.eslintignore --ext .js,.jsx,.ts,.tsx . && tsc --noEmit" }, "author": "Apache", "license": "Apache-2.0", Loading @@ -28,53 +28,82 @@ "flask" ], "devDependencies": { "babel": "^6.23.0", "babel-core": "^6.26.3", "babel-eslint": "^10.1.0", "@babel/core": "^7.18.5", "@babel/eslint-parser": "^7.18.2", "@babel/plugin-transform-runtime": "^7.16.0", "@babel/preset-env": "^7.16.0", "@babel/preset-react": "^7.16.0", "@babel/preset-typescript": "^7.17.12", "@testing-library/jest-dom": "^5.16.0", "@testing-library/react": "^13.0.0", "@types/react": "^18.0.12", "@types/react-dom": "^18.0.5", "@typescript-eslint/eslint-plugin": "^5.13.0", "@typescript-eslint/parser": "^5.0.0", "babel-jest": "^27.3.1", "babel-loader": "^8.1.0", "babel-plugin-css-modules-transform": "^1.6.1", "babel-polyfill": "^6.26.0", "clean-webpack-plugin": "^3.0.0", "copy-webpack-plugin": "^6.0.3", "css-loader": "^3.4.2", "eslint": "^7.5.0", "eslint-config-airbnb-base": "^14.2.0", "css-loader": "5.2.7", "css-minimizer-webpack-plugin": "^4.0.0", "eslint": "^8.6.0", "eslint-config-airbnb": "^19.0.4", "eslint-config-airbnb-typescript": "^17.0.0", "eslint-plugin-html": "^6.0.2", "eslint-plugin-import": "^2.22.0", "eslint-plugin-import": "^2.25.3", "eslint-plugin-jsx-a11y": "^6.5.0", "eslint-plugin-node": "^11.1.0", "eslint-plugin-promise": "^4.2.1", "eslint-plugin-react": "^7.30.0", "eslint-plugin-react-hooks": "^4.5.0", "eslint-plugin-standard": "^4.0.1", "file-loader": "^6.0.0", "imports-loader": "^1.1.0", "mini-css-extract-plugin": "1.6.0", "jest": "^27.3.1", "mini-css-extract-plugin": "^1.6.2", "moment": "^2.29.3", "moment-locales-webpack-plugin": "^1.2.0", "optimize-css-assets-webpack-plugin": "6.0.0", "nock": "^13.2.4", "style-loader": "^1.2.1", "stylelint": "^13.6.1", "stylelint-config-standard": "^20.0.0", "terser-webpack-plugin": "<5.0.0", "typescript": "^4.6.3", "url-loader": "4.1.0", "webpack": "^4.16.3", "webpack-cli": "^3.1.0", "webpack-manifest-plugin": "^2.2.0" "webpack": "^5.73.0", "webpack-cli": "^4.0.0", "webpack-license-plugin": "^4.2.1", "webpack-manifest-plugin": "^4.0.0" }, "dependencies": { "@chakra-ui/react": "^2.2.0", "@emotion/cache": "^11.9.3", "@emotion/react": "^11.9.3", "@emotion/styled": "^11", "axios": "^0.26.0", "bootstrap-3-typeahead": "^4.0.2", "camelcase-keys": "^7.0.0", "codemirror": "^5.59.1", "d3": "^3.4.4", "d3-shape": "^2.1.0", "d3-tip": "^0.9.1", "dagre-d3": "^0.6.4", "datatables.net": "^1.10.23", "datatables.net-bs": "^1.10.23", "datatables.net": "^1.11.4", "datatables.net-bs": "^1.11.4", "eonasdan-bootstrap-datetimepicker": "^4.17.47", "jquery": ">=3.4.0", "jshint": "^2.12.0", "moment-timezone": "^0.5.28", "framer-motion": "^6.0.0", "jquery": ">=3.5.0", "jshint": "^2.13.4", "lodash": "^4.17.21", "moment-timezone": "^0.5.34", "nvd3": "^1.8.6", "redoc": "^2.0.0-rc.48", "react": "^18.0.0", "react-dom": "^18.0.0", "react-icons": "^4.3.1", "react-query": "^3.39.1", "react-router-dom": "^6.3.0", "react-table": "^7.8.0", "redoc": "^2.0.0-rc.72", "url-search-params-polyfill": "^8.1.0" }, "resolutions": { "lodash": "^4.17.21" } }
pkgs/development/python-modules/apache-airflow/providers.nix 0 → 100644 +303 −0 File added.Preview size limit exceeded, changes collapsed. Show changes
pkgs/development/python-modules/apache-airflow/update-providers.py 0 → 100755 +226 −0 Original line number Diff line number Diff line #! /usr/bin/env python3 from itertools import chain import json import logging from pathlib import Path import os import re import subprocess import sys from typing import Dict, List, Optional, Set, TextIO from urllib.request import urlopen from urllib.error import HTTPError import yaml PKG_SET = "pkgs.python3Packages" # If some requirements are matched by multiple or no Python packages, the # following can be used to choose the correct one PKG_PREFERENCES = { "dnspython": "dnspython", "google-api-python-client": "google-api-python-client", "psycopg2-binary": "psycopg2", "requests_toolbelt": "requests-toolbelt", } # Requirements missing from the airflow provider metadata EXTRA_REQS = { "sftp": ["pysftp"], } def get_version(): with open(os.path.dirname(sys.argv[0]) + "/default.nix") as fh: # A version consists of digits, dots, and possibly a "b" (for beta) m = re.search('version = "([\\d\\.b]+)";', fh.read()) return m.group(1) def get_file_from_github(version: str, path: str): with urlopen( f"https://raw.githubusercontent.com/apache/airflow/{version}/{path}" ) as response: return yaml.safe_load(response) def repository_root() -> Path: return Path(os.path.dirname(sys.argv[0])) / "../../../.." def dump_packages() -> Dict[str, Dict[str, str]]: # Store a JSON dump of Nixpkgs' python3Packages output = subprocess.check_output( [ "nix-env", "-f", repository_root(), "-qa", "-A", PKG_SET, "--arg", "config", "{ allowAliases = false; }", "--json", ] ) return json.loads(output) def remove_version_constraint(req: str) -> str: return re.sub(r"[=><~].*$", "", req) def name_to_attr_path(req: str, packages: Dict[str, Dict[str, str]]) -> Optional[str]: if req in PKG_PREFERENCES: return f"{PKG_SET}.{PKG_PREFERENCES[req]}" attr_paths = [] names = [req] # E.g. python-mpd2 is actually called python3.6-mpd2 # instead of python-3.6-python-mpd2 inside Nixpkgs if req.startswith("python-") or req.startswith("python_"): names.append(req[len("python-") :]) for name in names: # treat "-" and "_" equally name = re.sub("[-_]", "[-_]", name) # python(minor).(major)-(pname)-(version or unstable-date) # we need the version qualifier, or we'll have multiple matches # (e.g. pyserial and pyserial-asyncio when looking for pyserial) pattern = re.compile( f"^python\\d+\\.\\d+-{name}-(?:\\d|unstable-.*)", re.I ) for attr_path, package in packages.items(): # logging.debug("Checking match for %s with %s", name, package["name"]) if pattern.match(package["name"]): attr_paths.append(attr_path) # Let's hope there's only one derivation with a matching name assert len(attr_paths) <= 1, f"{req} matches more than one derivation: {attr_paths}" if attr_paths: return attr_paths[0] return None def provider_reqs_to_attr_paths(reqs: List, packages: Dict) -> List: no_version_reqs = map(remove_version_constraint, reqs) filtered_reqs = [ req for req in no_version_reqs if not re.match(r"^apache-airflow", req) ] attr_paths = [] for req in filtered_reqs: attr_path = name_to_attr_path(req, packages) if attr_path is not None: # Add attribute path without "python3Packages." prefix pname = attr_path[len(PKG_SET + ".") :] attr_paths.append(pname) else: # If we can't find it, we just skip and warn the user logging.warning("Could not find package attr for %s", req) return attr_paths def get_cross_provider_reqs( provider: str, provider_reqs: Dict, cross_provider_deps: Dict, seen: List = None ) -> Set: # Unfortunately there are circular cross-provider dependencies, so keep a # list of ones we've seen already seen = seen or [] reqs = set(provider_reqs[provider]) if len(cross_provider_deps[provider]) > 0: reqs.update( chain.from_iterable( get_cross_provider_reqs( d, provider_reqs, cross_provider_deps, seen + [provider] ) if d not in seen else [] for d in cross_provider_deps[provider] ) ) return reqs def get_provider_reqs(version: str, packages: Dict) -> Dict: provider_dependencies = get_file_from_github( version, "generated/provider_dependencies.json" ) provider_reqs = {} cross_provider_deps = {} for provider, provider_data in provider_dependencies.items(): provider_reqs[provider] = list( provider_reqs_to_attr_paths(provider_data["deps"], packages) ) + EXTRA_REQS.get(provider, []) cross_provider_deps[provider] = [ d for d in provider_data["cross-providers-deps"] if d != "common.sql" ] transitive_provider_reqs = {} # Add transitive cross-provider reqs for provider in provider_reqs: transitive_provider_reqs[provider] = get_cross_provider_reqs( provider, provider_reqs, cross_provider_deps ) return transitive_provider_reqs def get_provider_yaml(version: str, provider: str) -> Dict: provider_dir = provider.replace(".", "/") path = f"airflow/providers/{provider_dir}/provider.yaml" try: return get_file_from_github(version, path) except HTTPError: logging.warning("Couldn't get provider yaml for %s", provider) return {} def get_provider_imports(version: str, providers) -> Dict: provider_imports = {} for provider in providers: provider_yaml = get_provider_yaml(version, provider) imports: List[str] = [] if "hooks" in provider_yaml: imports.extend( chain.from_iterable( hook["python-modules"] for hook in provider_yaml["hooks"] ) ) if "operators" in provider_yaml: imports.extend( chain.from_iterable( operator["python-modules"] for operator in provider_yaml["operators"] ) ) provider_imports[provider] = imports return provider_imports def to_nix_expr(provider_reqs: Dict, provider_imports: Dict, fh: TextIO) -> None: fh.write("# Warning: generated by update-providers.py, do not update manually\n") fh.write("{\n") for provider, reqs in provider_reqs.items(): provider_name = provider.replace(".", "_") fh.write(f" {provider_name} = {{\n") fh.write( " deps = [ " + " ".join(sorted(f'"{req}"' for req in reqs)) + " ];\n" ) fh.write( " imports = [ " + " ".join(sorted(f'"{imp}"' for imp in provider_imports[provider])) + " ];\n" ) fh.write(" };\n") fh.write("}\n") def main() -> None: logging.basicConfig(level=logging.INFO) version = get_version() packages = dump_packages() logging.info("Generating providers.nix for version %s", version) provider_reqs = get_provider_reqs(version, packages) provider_imports = get_provider_imports(version, provider_reqs.keys()) with open("providers.nix", "w") as fh: to_nix_expr(provider_reqs, provider_imports, fh) if __name__ == "__main__": main()
