apache-airflow: 2.10.5 -> 3.1.6 (#434461)
This commit is contained in:
Wolfgang Walther
@@ -3,276 +3,41 @@
|
||||
fetchpatch,
|
||||
fetchPypi,
|
||||
python3,
|
||||
|
||||
# Extra airflow providers to enable
|
||||
enabledProviders ? [ ],
|
||||
}:
|
||||
|
||||
let
|
||||
python = python3.override {
|
||||
self = python;
|
||||
packageOverrides = pySelf: pySuper: {
|
||||
connexion = pySuper.connexion.overridePythonAttrs rec {
|
||||
version = "2.14.2";
|
||||
src = fetchFromGitHub {
|
||||
owner = "spec-first";
|
||||
repo = "connexion";
|
||||
tag = version;
|
||||
hash = "sha256-1v1xCHY3ZnZG/Vu9wN/it7rLKC/StoDefoMNs+hMjIs=";
|
||||
};
|
||||
nativeBuildInputs = with pySelf; [
|
||||
setuptools
|
||||
];
|
||||
pythonRelaxDeps = [
|
||||
"werkzeug"
|
||||
];
|
||||
propagatedBuildInputs = with pySelf; [
|
||||
aiohttp
|
||||
aiohttp-jinja2
|
||||
aiohttp-swagger
|
||||
clickclick
|
||||
flask
|
||||
inflection
|
||||
jsonschema
|
||||
openapi-spec-validator
|
||||
packaging
|
||||
pyyaml
|
||||
requests
|
||||
swagger-ui-bundle
|
||||
];
|
||||
nativeCheckInputs = with pySelf; [
|
||||
aiohttp-remotes
|
||||
decorator
|
||||
pytest-aiohttp
|
||||
pytest-tornasync
|
||||
pytestCheckHook
|
||||
testfixtures
|
||||
];
|
||||
disabledTests = [
|
||||
"test_aiohttp_simple_api"
|
||||
"test_app"
|
||||
"test_invalid_type" # https://github.com/spec-first/connexion/issues/1969
|
||||
"test_openapi_yaml_behind_proxy"
|
||||
"test_run_with_wsgi_containers"
|
||||
"test_swagger_ui"
|
||||
];
|
||||
postPatch = ''
|
||||
substituteInPlace connexion/__init__.py \
|
||||
--replace "2020.0.dev1" "${version}"
|
||||
'';
|
||||
};
|
||||
werkzeug = pySuper.werkzeug.overridePythonAttrs rec {
|
||||
version = "2.3.8";
|
||||
src = fetchPypi {
|
||||
pname = "werkzeug";
|
||||
inherit version;
|
||||
hash = "sha256-VUslfHS763oNJUFgpPj/4YUkP1KlIDUGC3Ycpi2XfwM=";
|
||||
};
|
||||
nativeCheckInputs = with pySelf; [
|
||||
pytest-xprocess
|
||||
];
|
||||
};
|
||||
# flask's test-suite needs click 8.1.8
|
||||
# TypeError: CliRunner.__init__() got an unexpected keyword argument 'mix_stderr'
|
||||
click = pySuper.click.overridePythonAttrs rec {
|
||||
version = "8.1.8";
|
||||
src = fetchPypi {
|
||||
pname = "click";
|
||||
inherit version;
|
||||
hash = "sha256-7VPJ2JkNg8Kifermjk7jN0c/YzDAQKMdQiXJV00WCWo=";
|
||||
};
|
||||
};
|
||||
flask = pySuper.flask.overridePythonAttrs (o: rec {
|
||||
version = "2.2.5";
|
||||
src = fetchPypi {
|
||||
pname = "Flask";
|
||||
inherit version;
|
||||
hash = "sha256-7e6bCn/yZiG9WowQ/0hK4oc3okENmbC7mmhQx/uXeqA=";
|
||||
};
|
||||
nativeBuildInputs = (o.nativeBuildInputs or [ ]) ++ [
|
||||
pySelf.setuptools
|
||||
];
|
||||
pytestFlagsArray = [
|
||||
# tests that are marked with filterwarnings fail with
|
||||
# DeprecationWarning: 'pkgutil.get_loader' is deprecated and slated for
|
||||
# removal in Python 3.14; use importlib.util.find_spec() instead
|
||||
"-W ignore::DeprecationWarning"
|
||||
];
|
||||
});
|
||||
flask-login = pySuper.flask-login.overridePythonAttrs rec {
|
||||
version = "0.6.3";
|
||||
src = fetchFromGitHub {
|
||||
owner = "maxcountryman";
|
||||
repo = "flask-login";
|
||||
tag = version;
|
||||
hash = "sha256-Sn7Ond67P/3+OmKKFE/KfA6FE4IajhiRXVVrXKJtY3I=";
|
||||
};
|
||||
nativeBuildInputs = with pySelf; [ setuptools ];
|
||||
pytestFlagsArray = [
|
||||
# DeprecationWarning: datetime.datetime.utcnow() is deprecated
|
||||
# and scheduled for removal in a future version.
|
||||
# Use timezone-aware objects to represent datetimes in UTC:
|
||||
# datetime.datetime.now(datetime.UTC).
|
||||
"-W ignore::DeprecationWarning"
|
||||
];
|
||||
};
|
||||
flask-session = pySuper.flask-session.overridePythonAttrs rec {
|
||||
version = "0.5.0";
|
||||
src = fetchFromGitHub {
|
||||
owner = "palletc-eco";
|
||||
repo = "flask-session";
|
||||
tag = version;
|
||||
hash = "sha256-t8w6ZS4gBDpnnKvL3DLtn+rRLQNJbrT2Hxm4f3+a3Xc=";
|
||||
};
|
||||
nativeCheckInputs = with pySelf; [ pytestCheckHook ];
|
||||
pytestFlagsArray = [
|
||||
"-k"
|
||||
"'null_session or filesystem_session'"
|
||||
];
|
||||
dependencies = with pySelf; [
|
||||
flask-sqlalchemy
|
||||
cachelib
|
||||
];
|
||||
disabledTests = [ ];
|
||||
disabledTestPaths = [ ];
|
||||
preCheck = "";
|
||||
postCheck = "";
|
||||
};
|
||||
# flask-appbuilder doesn't work with sqlalchemy 2.x, flask-appbuilder 3.x
|
||||
# https://github.com/dpgaspar/Flask-AppBuilder/issues/2038
|
||||
flask-appbuilder = pySuper.flask-appbuilder.overridePythonAttrs {
|
||||
meta.broken = false;
|
||||
};
|
||||
# a knock-on effect from overriding the sqlalchemy version
|
||||
flask-sqlalchemy = pySuper.flask-sqlalchemy.overridePythonAttrs {
|
||||
src = fetchPypi {
|
||||
pname = "Flask-SQLAlchemy";
|
||||
version = "2.5.1";
|
||||
hash = "sha256-K9pEtD58rLFdTgX/PMH4vJeTbMRkYjQkECv8LDXpWRI=";
|
||||
};
|
||||
format = "setuptools";
|
||||
};
|
||||
httpcore = pySuper.httpcore.overridePythonAttrs {
|
||||
# nullify upstream's pytest flags which cause
|
||||
# "TLS/SSL connection has been closed (EOF)"
|
||||
# with pytest-httpbin 1.x
|
||||
preCheck = ''
|
||||
substituteInPlace pyproject.toml \
|
||||
--replace '[tool.pytest.ini_options]' '[tool.notpytest.ini_options]'
|
||||
'';
|
||||
};
|
||||
pytest-httpbin = pySuper.pytest-httpbin.overridePythonAttrs rec {
|
||||
version = "1.0.2";
|
||||
src = fetchFromGitHub {
|
||||
owner = "kevin1024";
|
||||
repo = "pytest-httpbin";
|
||||
tag = "v${version}";
|
||||
hash = "sha256-S4ThQx4H3UlKhunJo35esPClZiEn7gX/Qwo4kE1QMTI=";
|
||||
};
|
||||
# Use unmerged PR #65 to fix older version:
|
||||
# https://github.com/kevin1024/pytest-httpbin/pull/65/
|
||||
# It was closed in favour of another which isn't compatible with the overriden version.
|
||||
patches = [
|
||||
(fetchpatch {
|
||||
url = "https://github.com/kevin1024/pytest-httpbin/commit/4e325f877ff8f77dec9f380bd8e53bb42976775c.patch";
|
||||
hash = "sha256-a33XcdMupD+7ZzvUibePdldGImmPLDNU2sxRbwpveDA=";
|
||||
})
|
||||
(fetchpatch {
|
||||
url = "https://github.com/kevin1024/pytest-httpbin/commit/463afb9b200563ac6fe7ae535f7a7a3c818b0418.patch";
|
||||
hash = "sha256-HFmuLtAtEjnB6heSG1YNnqxtz2phXNkHbQaZyB5bLJs=";
|
||||
})
|
||||
];
|
||||
disabledTests = [
|
||||
"test_httpbin_secure_accepts_get_requests"
|
||||
"test_httpbin_secure_accepts_lots_of_get_requests"
|
||||
"test_httpbin_both[https]"
|
||||
"test_chunked_encoding[https]"
|
||||
"TestClassBassedTests::test_http_secure"
|
||||
"test_dont_crash_on_certificate_problems"
|
||||
"test_redirect_location_is_https_for_secure_server"
|
||||
"test_httpbin_secure_accepts_get_requests"
|
||||
"test_http_secure"
|
||||
];
|
||||
};
|
||||
# apache-airflow doesn't work with sqlalchemy 2.x
|
||||
# https://github.com/apache/airflow/issues/28723
|
||||
sqlalchemy = pySuper.sqlalchemy_1_4;
|
||||
gitpython = pySuper.gitpython.overridePythonAttrs rec {
|
||||
version = "3.1.44";
|
||||
src = fetchFromGitHub {
|
||||
owner = "gitpython-developers";
|
||||
repo = "gitpython";
|
||||
rev = version;
|
||||
hash = "sha256-KnKaBv/tKk4wiGWUWCEgd1vgrTouwUhqxJ1/nMjRaWk=";
|
||||
};
|
||||
};
|
||||
# ValueError: Unknown classifier in field `project.classifiers`: Programming Language :: Python :: Free Threading :: 2 - Beta
|
||||
urllib3 = pySuper.urllib3.overridePythonAttrs rec {
|
||||
version = "2.5.0";
|
||||
src = fetchPypi {
|
||||
pname = "urllib3";
|
||||
inherit version;
|
||||
hash = "sha256-P8R3M8fkGdS8P2s9wrT4kLt0OQajDVa6Slv6S7/5J2A=";
|
||||
};
|
||||
postPatch = ''
|
||||
substituteInPlace pyproject.toml \
|
||||
--replace-fail ', "setuptools-scm>=8,<9"' ""
|
||||
'';
|
||||
};
|
||||
smmap = pySuper.smmap.overridePythonAttrs rec {
|
||||
version = "5.0.2";
|
||||
src = fetchFromGitHub {
|
||||
owner = "gitpython-developers";
|
||||
repo = "smmap";
|
||||
rev = "refs/tags/v${version}";
|
||||
tag = "v${version}";
|
||||
hash = "sha256-0Y175kjv/8UJpSxtLpWH4/VT7JrcVPAq79Nf3rtHZZM=";
|
||||
};
|
||||
};
|
||||
trove-classifiers = pySuper.trove-classifiers.overridePythonAttrs rec {
|
||||
version = "2024.10.21.16";
|
||||
src = fetchPypi {
|
||||
pname = "trove_classifiers";
|
||||
inherit version;
|
||||
hash = "sha256-F8vQVdZ9Xp2d5jKTqHMpQ/q8IVdOTHt07fEStJKM9fM=";
|
||||
};
|
||||
postPatch = "";
|
||||
};
|
||||
packaging = pySuper.packaging.overridePythonAttrs rec {
|
||||
version = "24.2";
|
||||
src = fetchPypi {
|
||||
pname = "packaging";
|
||||
inherit version;
|
||||
hash = "sha256-wiim3F6TLTRrxXOTeRCdSeiFPdgiNXHHxbVSYO3AuX8=";
|
||||
};
|
||||
};
|
||||
pluggy = pySuper.pluggy.overridePythonAttrs rec {
|
||||
version = "1.5.0";
|
||||
structlog = pySuper.structlog.overridePythonAttrs (o: rec {
|
||||
version = "25.4.0";
|
||||
src = fetchFromGitHub {
|
||||
owner = "pytest-dev";
|
||||
repo = "pluggy";
|
||||
owner = "hynek";
|
||||
repo = "structlog";
|
||||
tag = version;
|
||||
hash = "sha256-f0DxyZZk6RoYtOEXLACcsOn2B+Hot4U4g5Ogr/hKmOE=";
|
||||
hash = "sha256-iNnUogcICQJvHBZO2J8uk4NleQY/ra3ZzxQgnSRKr30=";
|
||||
};
|
||||
};
|
||||
pyproject-api = pySuper.pyproject-api.overridePythonAttrs rec {
|
||||
version = "1.8.0";
|
||||
src = fetchPypi {
|
||||
pname = "pyproject_api";
|
||||
inherit version;
|
||||
hash = "sha256-d7gEny/rXTPu/MIbV/HieWNid6isita1hxA3skN3hJY=";
|
||||
};
|
||||
disabledTests = [
|
||||
# AssertionError: assert ['magic>3', 'requests>2'] == ['magic >3', 'requests >2']
|
||||
"test_frontend_setuptools"
|
||||
];
|
||||
};
|
||||
tox = pySuper.tox.overridePythonAttrs rec {
|
||||
version = "4.27.0";
|
||||
src = fetchFromGitHub {
|
||||
owner = "tox-dev";
|
||||
repo = "tox";
|
||||
tag = version;
|
||||
hash = "sha256-Z3qUK4w1ebPvdZD4ZuKgZXJPUu5lG0G41vn/pc9gC/0=";
|
||||
};
|
||||
};
|
||||
apache-airflow = pySelf.callPackage ./python-package.nix { };
|
||||
nativeCheckInputs =
|
||||
with pySelf;
|
||||
o.nativeCheckInputs
|
||||
++ [
|
||||
freezegun
|
||||
pretend
|
||||
];
|
||||
});
|
||||
apache-airflow = pySelf.callPackage ./python-package.nix { inherit enabledProviders; };
|
||||
};
|
||||
};
|
||||
in
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -2,115 +2,156 @@
|
||||
lib,
|
||||
stdenv,
|
||||
fetchFromGitHub,
|
||||
writeScript,
|
||||
writableTmpDirAsHomeHook,
|
||||
|
||||
# javascript
|
||||
fetchYarnDeps,
|
||||
fetchPnpmDeps,
|
||||
nodejs,
|
||||
webpack-cli,
|
||||
yarnBuildHook,
|
||||
yarnConfigHook,
|
||||
pnpm,
|
||||
pnpmConfigHook,
|
||||
|
||||
# python
|
||||
a2wsgi,
|
||||
aiosqlite,
|
||||
alembic,
|
||||
argcomplete,
|
||||
asgiref,
|
||||
attrs,
|
||||
babel,
|
||||
buildPythonPackage,
|
||||
cadwyn,
|
||||
colorlog,
|
||||
configupdater,
|
||||
connexion,
|
||||
cron-descriptor,
|
||||
croniter,
|
||||
cryptography,
|
||||
deprecated,
|
||||
dill,
|
||||
flask-caching,
|
||||
flask-login,
|
||||
flask-session,
|
||||
fastapi,
|
||||
flit-core,
|
||||
fsspec,
|
||||
gitdb,
|
||||
gitpython,
|
||||
gunicorn,
|
||||
greenback,
|
||||
hatchling,
|
||||
httpx,
|
||||
importlib-metadata,
|
||||
itsdangerous,
|
||||
jinja2,
|
||||
jsonschema,
|
||||
lazy-object-proxy,
|
||||
libcst,
|
||||
linkify-it-py,
|
||||
lockfile,
|
||||
marshmallow-oneofschema,
|
||||
mdit-py-plugins,
|
||||
methodtools,
|
||||
msgspec,
|
||||
natsort,
|
||||
opentelemetry-api,
|
||||
opentelemetry-exporter-otlp,
|
||||
packaging,
|
||||
pandas,
|
||||
pathspec,
|
||||
pendulum,
|
||||
pluggy,
|
||||
psutil,
|
||||
pytest-asyncio,
|
||||
pytestCheckHook,
|
||||
pydantic,
|
||||
pygments,
|
||||
pygtrie,
|
||||
pyjwt,
|
||||
python,
|
||||
python-daemon,
|
||||
python-nvd3,
|
||||
python-dateutil,
|
||||
python-slugify,
|
||||
pyyaml,
|
||||
requests,
|
||||
retryhttp,
|
||||
rich,
|
||||
rich-argparse,
|
||||
rich-click,
|
||||
setproctitle,
|
||||
smmap,
|
||||
sqlalchemy,
|
||||
sqlalchemy-jsonfield,
|
||||
sqlalchemy-utils,
|
||||
starlette,
|
||||
structlog,
|
||||
svcs,
|
||||
tabulate,
|
||||
tenacity,
|
||||
termcolor,
|
||||
tomli,
|
||||
trove-classifiers,
|
||||
types-requests,
|
||||
typing-extensions,
|
||||
universal-pathlib,
|
||||
uuid6,
|
||||
uvicorn,
|
||||
|
||||
# Extra airflow providers to enable
|
||||
enabledProviders ? [ ],
|
||||
enabledProviders,
|
||||
}:
|
||||
let
|
||||
version = "2.10.5";
|
||||
version = "3.1.6";
|
||||
|
||||
airflow-src = fetchFromGitHub {
|
||||
src = fetchFromGitHub {
|
||||
owner = "apache";
|
||||
repo = "airflow";
|
||||
tag = version;
|
||||
# Download using the git protocol rather than using tarballs, because the
|
||||
# GitHub archive tarballs don't appear to include tests
|
||||
forceFetchGit = true;
|
||||
hash = "sha256-q5/CM+puXE31+15F3yZmcrR74LrqHppdCDUqjLQXPfk=";
|
||||
hash = "sha256-wC6C0jhCA76/+KhBQbe3WeSGqR6FwaudCT5xPV39Z6c=";
|
||||
};
|
||||
|
||||
# airflow bundles a web interface, which is built using webpack by an undocumented shell script in airflow's source tree.
|
||||
# This replicates this shell script, fixing bugs in yarn.lock and package.json
|
||||
|
||||
airflow-frontend = stdenv.mkDerivation rec {
|
||||
name = "airflow-frontend";
|
||||
|
||||
src = "${airflow-src}/airflow/www";
|
||||
|
||||
offlineCache = fetchYarnDeps {
|
||||
yarnLock = "${src}/yarn.lock";
|
||||
hash = "sha256-hKgtMH4c8sPRDLPLVn+H8rmwc2Q6ei6U4er6fGuFn4I=";
|
||||
};
|
||||
airflowUi = stdenv.mkDerivation rec {
|
||||
pname = "airflow-ui-assets";
|
||||
inherit src version;
|
||||
sourceRoot = "${src.name}/airflow-core/src/airflow/ui";
|
||||
|
||||
nativeBuildInputs = [
|
||||
yarnConfigHook
|
||||
yarnBuildHook
|
||||
nodejs
|
||||
webpack-cli
|
||||
pnpm
|
||||
pnpmConfigHook
|
||||
];
|
||||
|
||||
# The webpack license plugin tries to create /3rd-party-licenses when given the
|
||||
# original relative path
|
||||
postPatch = ''
|
||||
sed -i 's!../../../../3rd-party-licenses/LICENSES-ui.txt!/3rd-party-licenses/LICENSES-ui.txt!' webpack.config.js
|
||||
'';
|
||||
pnpmDeps = fetchPnpmDeps {
|
||||
pname = "airflow-ui";
|
||||
inherit sourceRoot src version;
|
||||
fetcherVersion = 1;
|
||||
hash = "sha256-UcEFQkDZ9Ye+VfyJ9rdZKe0wilTgO4dMsULABWfL2Co=";
|
||||
};
|
||||
|
||||
postBuild = ''
|
||||
find package.json yarn.lock static/css static/js -type f | sort | xargs md5sum > static/dist/sum.md5
|
||||
buildPhase = ''
|
||||
pnpm install
|
||||
pnpm build
|
||||
'';
|
||||
|
||||
installPhase = ''
|
||||
mkdir -p $out/static/
|
||||
cp -r static/dist $out/static
|
||||
mkdir -p $out/share/airflow/ui
|
||||
cp -r dist $out/share/airflow/ui/
|
||||
'';
|
||||
};
|
||||
|
||||
airflowSimpleAuthUi = stdenv.mkDerivation rec {
|
||||
pname = "airflow-simple-ui-assets";
|
||||
inherit src version;
|
||||
sourceRoot = "${src.name}/airflow-core/src/airflow/api_fastapi/auth/managers/simple/ui";
|
||||
|
||||
nativeBuildInputs = [
|
||||
nodejs
|
||||
pnpm
|
||||
pnpmConfigHook
|
||||
];
|
||||
|
||||
pnpmDeps = fetchPnpmDeps {
|
||||
pname = "simple-auth-manager-ui";
|
||||
inherit sourceRoot src version;
|
||||
fetcherVersion = 1;
|
||||
hash = "sha256-8nZdWnhERUkiaY8USyy/a/j+dMksjmEzCabSkysndSE=";
|
||||
};
|
||||
|
||||
buildPhase = ''
|
||||
pnpm install
|
||||
pnpm build
|
||||
'';
|
||||
|
||||
installPhase = ''
|
||||
mkdir -p $out/share/airflow/simple-ui
|
||||
cp -r dist $out/share/airflow/simple-ui/
|
||||
'';
|
||||
};
|
||||
|
||||
@@ -118,216 +159,231 @@ let
|
||||
"common_compat"
|
||||
"common_io"
|
||||
"common_sql"
|
||||
"fab"
|
||||
"ftp"
|
||||
"http"
|
||||
"imap"
|
||||
"smtp"
|
||||
"sqlite"
|
||||
"standard"
|
||||
];
|
||||
|
||||
# Import generated file with metadata for provider dependencies and imports.
|
||||
# Enable additional providers using enabledProviders above.
|
||||
providers = import ./providers.nix;
|
||||
getProviderPath = provider: lib.replaceStrings [ "_" ] [ "/" ] provider;
|
||||
getProviderDeps = provider: map (dep: python.pkgs.${dep}) providers.${provider}.deps;
|
||||
getProviderImports = provider: providers.${provider}.imports;
|
||||
providerImports = lib.concatMap getProviderImports enabledProviders;
|
||||
|
||||
buildProvider =
|
||||
provider:
|
||||
let
|
||||
providerPath = getProviderPath provider;
|
||||
in
|
||||
python.pkgs.buildPythonPackage {
|
||||
buildPythonPackage {
|
||||
pname = "apache-airflow-providers-${provider}";
|
||||
version = "unstable"; # will be extracted in the build phase
|
||||
pyproject = false; # providers packages don't have pyproject.toml nor setup.py
|
||||
version = providers.${provider}.version;
|
||||
pyproject = true;
|
||||
|
||||
src = airflow-src;
|
||||
inherit src;
|
||||
sourceRoot = "${src.name}/providers/${lib.replaceStrings [ "_" ] [ "/" ] provider}";
|
||||
|
||||
propagatedBuildInputs = getProviderDeps provider;
|
||||
dependencies = [ packaging ];
|
||||
buildInputs = [ flit-core ];
|
||||
|
||||
buildPhase = ''
|
||||
# extract version from the provider's __init__.py file
|
||||
if [ -f "airflow/providers/${providerPath}/__init__.py" ]; then
|
||||
version=$(grep -oP "(?<=__version__ = ')[^']+" "airflow/providers/${providerPath}/__init__.py" || echo "0.0.0")
|
||||
echo "Provider ${provider} version: $version"
|
||||
else
|
||||
echo "Error: __init__.py not found for provider ${provider} at path airflow/providers/${providerPath}"
|
||||
exit 1
|
||||
fi
|
||||
'';
|
||||
dependencies = map (dep: python.pkgs.${dep}) providers.${provider}.deps;
|
||||
|
||||
installPhase = ''
|
||||
# create directory structure
|
||||
mkdir -p $out/${python.sitePackages}/airflow/providers
|
||||
pythonRemoveDeps = [
|
||||
"apache-airflow"
|
||||
];
|
||||
|
||||
# copy the provider directory
|
||||
if [ -d "airflow/providers/${providerPath}" ]; then
|
||||
mkdir -p $out/${python.sitePackages}/airflow/providers/$(dirname "${providerPath}")
|
||||
cp -r airflow/providers/${providerPath} $out/${python.sitePackages}/airflow/providers/$(dirname "${providerPath}")
|
||||
|
||||
# create parent __init__.py files
|
||||
touch $out/${python.sitePackages}/airflow/__init__.py
|
||||
touch $out/${python.sitePackages}/airflow/providers/__init__.py
|
||||
|
||||
# create any needed intermediate __init__.py files for nested providers
|
||||
providerDir=$(dirname "${providerPath}")
|
||||
while [ "$providerDir" != "." ] && [ -n "$providerDir" ]; do
|
||||
mkdir -p $out/${python.sitePackages}/airflow/providers/$providerDir
|
||||
touch $out/${python.sitePackages}/airflow/providers/$providerDir/__init__.py
|
||||
providerDir=$(dirname "$providerDir")
|
||||
done
|
||||
|
||||
# create egg-info for package discovery
|
||||
mkdir -p $out/${python.sitePackages}/apache_airflow_providers_${provider}.egg-info
|
||||
cat > $out/${python.sitePackages}/apache_airflow_providers_${provider}.egg-info/PKG-INFO <<EOF
|
||||
Metadata-Version: 2.1
|
||||
Name: apache-airflow-providers-${lib.replaceStrings [ "_" ] [ "-" ] provider}
|
||||
Version: ${version}
|
||||
Summary: Apache Airflow Provider for ${provider}
|
||||
EOF
|
||||
else
|
||||
echo "Provider directory not found: airflow/providers/${providerPath}"
|
||||
exit 1
|
||||
fi
|
||||
'';
|
||||
pythonRelaxDeps = [
|
||||
"flit-core"
|
||||
];
|
||||
};
|
||||
|
||||
providerPackages = map buildProvider (requiredProviders ++ enabledProviders);
|
||||
airflowCore = buildPythonPackage {
|
||||
pname = "apache-airflow-core";
|
||||
inherit src version;
|
||||
pyproject = true;
|
||||
|
||||
sourceRoot = "${src.name}/airflow-core";
|
||||
|
||||
postPatch = ''
|
||||
# remove cyclic dependency
|
||||
sed -i -E 's/"apache-airflow-task-sdk[^"]+",//' pyproject.toml
|
||||
|
||||
substituteInPlace pyproject.toml \
|
||||
--replace-fail "hatchling==1.27.0" "hatchling" \
|
||||
--replace-fail "trove-classifiers==2025.9.11.17" "trove-classifiers"
|
||||
|
||||
# Copy built UI assets
|
||||
cp -r ${airflowUi}/share/airflow/ui/dist src/airflow/ui/
|
||||
cp -r ${airflowSimpleAuthUi}/share/airflow/simple-ui/dist src/airflow/api_fastapi/auth/managers/simple/ui/
|
||||
'';
|
||||
|
||||
build-system = [
|
||||
gitdb
|
||||
gitpython
|
||||
hatchling
|
||||
packaging
|
||||
smmap
|
||||
tomli
|
||||
trove-classifiers
|
||||
];
|
||||
|
||||
dependencies = [
|
||||
a2wsgi
|
||||
aiosqlite
|
||||
alembic
|
||||
argcomplete
|
||||
asgiref
|
||||
attrs
|
||||
cadwyn
|
||||
colorlog
|
||||
cron-descriptor
|
||||
croniter
|
||||
cryptography
|
||||
deprecated
|
||||
dill
|
||||
fastapi
|
||||
httpx
|
||||
importlib-metadata
|
||||
itsdangerous
|
||||
jinja2
|
||||
jsonschema
|
||||
lazy-object-proxy
|
||||
libcst
|
||||
linkify-it-py
|
||||
lockfile
|
||||
methodtools
|
||||
msgspec
|
||||
natsort
|
||||
opentelemetry-api
|
||||
opentelemetry-exporter-otlp
|
||||
packaging
|
||||
pathspec
|
||||
pendulum
|
||||
pluggy
|
||||
psutil
|
||||
pydantic
|
||||
pygments
|
||||
pygtrie
|
||||
pyjwt
|
||||
python-daemon
|
||||
python-dateutil
|
||||
python-slugify
|
||||
pyyaml
|
||||
requests
|
||||
rich
|
||||
rich-argparse
|
||||
rich-click
|
||||
setproctitle
|
||||
sqlalchemy
|
||||
sqlalchemy-jsonfield
|
||||
sqlalchemy-utils
|
||||
starlette
|
||||
structlog
|
||||
svcs
|
||||
tabulate
|
||||
taskSdk
|
||||
tenacity
|
||||
termcolor
|
||||
typing-extensions
|
||||
universal-pathlib
|
||||
uuid6
|
||||
uvicorn
|
||||
]
|
||||
++ (map buildProvider requiredProviders);
|
||||
|
||||
pythonRelaxDeps = [
|
||||
# Temporary to fix CI only:
|
||||
# https://github.com/apache/airflow/commit/c474be9ff06cf16bf96f93de9a09e30ffc476bee
|
||||
"fastapi"
|
||||
];
|
||||
};
|
||||
|
||||
taskSdk = buildPythonPackage {
|
||||
pname = "task-sdk";
|
||||
inherit src version;
|
||||
pyproject = true;
|
||||
|
||||
sourceRoot = "${src.name}/task-sdk";
|
||||
|
||||
postPatch = ''
|
||||
# resolve cyclic dependency
|
||||
sed -i -E 's/"apache-airflow-core[^"]+",//' pyproject.toml
|
||||
'';
|
||||
|
||||
build-system = [
|
||||
hatchling
|
||||
];
|
||||
|
||||
dependencies = [
|
||||
asgiref
|
||||
attrs
|
||||
babel
|
||||
colorlog
|
||||
fsspec
|
||||
greenback
|
||||
httpx
|
||||
jinja2
|
||||
methodtools
|
||||
msgspec
|
||||
pendulum
|
||||
psutil
|
||||
pydantic
|
||||
pygtrie
|
||||
python-dateutil
|
||||
requests
|
||||
retryhttp
|
||||
structlog
|
||||
tenacity
|
||||
types-requests
|
||||
];
|
||||
};
|
||||
|
||||
in
|
||||
buildPythonPackage rec {
|
||||
pname = "apache-airflow";
|
||||
inherit version;
|
||||
src = airflow-src;
|
||||
inherit src version;
|
||||
pyproject = true;
|
||||
|
||||
nativeBuildInputs = [ hatchling ];
|
||||
|
||||
dependencies = [
|
||||
alembic
|
||||
argcomplete
|
||||
colorlog
|
||||
configupdater
|
||||
connexion
|
||||
cron-descriptor
|
||||
croniter
|
||||
cryptography
|
||||
dill
|
||||
flask-caching
|
||||
flask-login
|
||||
flask-session
|
||||
fsspec
|
||||
gitdb
|
||||
gitpython
|
||||
gunicorn
|
||||
lazy-object-proxy
|
||||
linkify-it-py
|
||||
lockfile
|
||||
mdit-py-plugins
|
||||
methodtools
|
||||
opentelemetry-api
|
||||
opentelemetry-exporter-otlp
|
||||
packaging
|
||||
pandas
|
||||
pathspec
|
||||
pendulum
|
||||
pluggy
|
||||
psutil
|
||||
python-daemon
|
||||
python-nvd3
|
||||
python-slugify
|
||||
rich-argparse
|
||||
setproctitle
|
||||
smmap
|
||||
sqlalchemy
|
||||
sqlalchemy-jsonfield
|
||||
tabulate
|
||||
tenacity
|
||||
termcolor
|
||||
tomli
|
||||
trove-classifiers
|
||||
universal-pathlib
|
||||
]
|
||||
++ providerPackages;
|
||||
|
||||
nativeCheckInputs = [
|
||||
pytest-asyncio
|
||||
pytestCheckHook
|
||||
marshmallow-oneofschema
|
||||
];
|
||||
|
||||
checkPhase = ''
|
||||
export PYTEST_ADDOPTS="--asyncio_default_fixture_loop_scope=cache"
|
||||
'';
|
||||
|
||||
postPatch = ''
|
||||
substituteInPlace pyproject.toml \
|
||||
--replace-fail "hatchling==1.27.0" "hatchling" \
|
||||
--replace-fail "\"/airflow/providers/\"," ""
|
||||
--replace-fail "trove-classifiers==2025.9.11.17" "trove-classifiers"
|
||||
'';
|
||||
|
||||
pythonRelaxDeps = [
|
||||
"apache-airflow-providers-fab" # fab provider package has wrong version
|
||||
"colorlog"
|
||||
"pathspec"
|
||||
nativeBuildInputs = [ writableTmpDirAsHomeHook ];
|
||||
|
||||
build-system = [
|
||||
gitdb
|
||||
gitpython
|
||||
hatchling
|
||||
packaging
|
||||
pathspec
|
||||
pluggy
|
||||
smmap
|
||||
tomli
|
||||
trove-classifiers
|
||||
];
|
||||
|
||||
# allow for gunicorn processes to have access to Python packages
|
||||
makeWrapperArgs = [
|
||||
"--prefix PYTHONPATH : $PYTHONPATH"
|
||||
];
|
||||
dependencies = [
|
||||
airflowCore # subpackage from airflow src
|
||||
taskSdk # subpackage from airflow src
|
||||
]
|
||||
++ (map buildProvider enabledProviders);
|
||||
|
||||
postInstall = ''
|
||||
cp -rv ${airflow-frontend}/static/dist $out/${python.sitePackages}/airflow/www/static
|
||||
# Needed for pythonImportsCheck below
|
||||
export HOME=$(mktemp -d)
|
||||
# Create a symlink to the airflow-core package
|
||||
mkdir -p $out/bin
|
||||
ln -s ${airflowCore}/bin/airflow $out/bin/airflow
|
||||
'';
|
||||
|
||||
installCheckPhase = ''
|
||||
runHook preInstallCheck
|
||||
|
||||
$out/bin/airflow version
|
||||
$out/bin/airflow db reset -y
|
||||
|
||||
runHook postInstallCheck
|
||||
'';
|
||||
|
||||
pythonImportsCheck = [
|
||||
"airflow"
|
||||
]
|
||||
++ providerImports;
|
||||
++ lib.concatMap (provider: providers.${provider}.imports) (requiredProviders ++ enabledProviders);
|
||||
|
||||
preCheck = ''
|
||||
export AIRFLOW_HOME=$HOME
|
||||
export AIRFLOW__CORE__UNIT_TEST_MODE=True
|
||||
export AIRFLOW_DB="$HOME/airflow.db"
|
||||
export PATH=$PATH:$out/bin
|
||||
'';
|
||||
|
||||
enabledTestPaths = [
|
||||
"tests/core/test_core.py"
|
||||
];
|
||||
|
||||
disabledTests = lib.optionals stdenv.hostPlatform.isDarwin [
|
||||
"bash_operator_kill" # psutil.AccessDenied
|
||||
];
|
||||
|
||||
# Updates yarn.lock and package.json
|
||||
passthru.updateScript = writeScript "update.sh" ''
|
||||
#!/usr/bin/env nix-shell
|
||||
#!nix-shell -i bash -p common-updater-scripts curl "python3.withPackages (ps: with ps; [ pyyaml ])" yarn2nix
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
# Get new version
|
||||
new_version="$(curl -s https://airflow.apache.org/docs/apache-airflow/stable/release_notes.html |
|
||||
grep -oE 'Airflow [0-9.]+' | head -1 | grep -oE '[0-9.]+')"
|
||||
update-source-version ${pname} "$new_version"
|
||||
|
||||
# Update frontend
|
||||
cd ./pkgs/servers/apache-airflow
|
||||
curl -O https://raw.githubusercontent.com/apache/airflow/$new_version/airflow/www/yarn.lock
|
||||
curl -O https://raw.githubusercontent.com/apache/airflow/$new_version/airflow/www/package.json
|
||||
yarn2nix > yarn.nix
|
||||
|
||||
# update provider dependencies
|
||||
./update-providers.py
|
||||
'';
|
||||
passthru.updateScript = ./update.sh;
|
||||
passthru.airflowUi = airflowUi;
|
||||
passthru.airflowSimpleAuthUi = airflowSimpleAuthUi;
|
||||
|
||||
# Note on testing the web UI:
|
||||
# You can (manually) test the web UI as follows:
|
||||
@@ -341,12 +397,13 @@ buildPythonPackage rec {
|
||||
# triggering the 'example_bash_operator' DAG and see if it reports success.
|
||||
|
||||
meta = {
|
||||
description = "Programmatically author, schedule and monitor data pipelines";
|
||||
description = "Platform to programmatically author, schedule and monitor workflows";
|
||||
homepage = "https://airflow.apache.org/";
|
||||
changelog = "https://airflow.apache.org/docs/apache-airflow/${version}/release_notes.html";
|
||||
license = lib.licenses.asl20;
|
||||
maintainers = with lib.maintainers; [
|
||||
taranarmo
|
||||
];
|
||||
mainProgram = "airflow";
|
||||
};
|
||||
}
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
#! /usr/bin/env python3
|
||||
#!/usr/bin/env nix-shell
|
||||
#!nix-shell -i python3 -p "python3.withPackages(ps: [ps.requests ps.tomli ps.packaging])"
|
||||
|
||||
from itertools import chain
|
||||
import json
|
||||
@@ -11,7 +12,8 @@ import sys
|
||||
from typing import Dict, List, Optional, Set, TextIO
|
||||
from urllib.request import urlopen
|
||||
from urllib.error import HTTPError
|
||||
import yaml
|
||||
from packaging.requirements import Requirement
|
||||
import tomli
|
||||
|
||||
PKG_SET = "apache-airflow.pythonPackages"
|
||||
|
||||
@@ -21,8 +23,9 @@ PKG_PREFERENCES = {
|
||||
"dnspython": "dnspython",
|
||||
"elasticsearch-dsl": "elasticsearch-dsl",
|
||||
"google-api-python-client": "google-api-python-client",
|
||||
"numpy": "numpy",
|
||||
"protobuf": "protobuf",
|
||||
"psycopg2-binary": "psycopg2",
|
||||
"pydantic": "pydantic",
|
||||
"requests_toolbelt": "requests-toolbelt",
|
||||
}
|
||||
|
||||
@@ -35,7 +38,7 @@ EXTRA_REQS = {
|
||||
def get_version():
|
||||
with open(os.path.dirname(sys.argv[0]) + "/python-package.nix") as fh:
|
||||
# A version consists of digits, dots, and possibly a "b" (for beta)
|
||||
m = re.search('version = "([\\d\\.b]+)";', fh.read())
|
||||
m = re.search(r'version = "([\d.b]+)";', fh.read())
|
||||
return m.group(1)
|
||||
|
||||
|
||||
@@ -43,11 +46,11 @@ def get_file_from_github(version: str, path: str):
|
||||
with urlopen(
|
||||
f"https://raw.githubusercontent.com/apache/airflow/{version}/{path}"
|
||||
) as response:
|
||||
return yaml.safe_load(response)
|
||||
return response.read()
|
||||
|
||||
|
||||
def repository_root() -> Path:
|
||||
return Path(os.path.dirname(sys.argv[0])) / "../../../.."
|
||||
return Path(os.path.dirname(sys.argv[0])) / "../../../../"
|
||||
|
||||
|
||||
def dump_packages() -> Dict[str, Dict[str, str]]:
|
||||
@@ -70,27 +73,33 @@ def dump_packages() -> Dict[str, Dict[str, str]]:
|
||||
|
||||
|
||||
def remove_version_constraint(req: str) -> str:
|
||||
return re.sub(r"[=><~].*$", "", req)
|
||||
parsed_req = Requirement(req)
|
||||
name = parsed_req.name
|
||||
extras = ",".join(sorted(parsed_req.extras))
|
||||
if extras:
|
||||
return f"{name}[{extras}]"
|
||||
return name
|
||||
|
||||
|
||||
def name_to_attr_path(req: str, packages: Dict[str, Dict[str, str]]) -> Optional[str]:
|
||||
if req in PKG_PREFERENCES:
|
||||
return f"{PKG_SET}.{PKG_PREFERENCES[req]}"
|
||||
# Extract the base name, removing any extras (e.g., '[flask]')
|
||||
base_req_name = req.split("[")[0]
|
||||
logging.debug(f"Searching for base_req_name: {base_req_name}")
|
||||
if base_req_name in PKG_PREFERENCES:
|
||||
return f"{PKG_SET}.{PKG_PREFERENCES[base_req_name]}"
|
||||
attr_paths = []
|
||||
names = [req]
|
||||
names = [base_req_name]
|
||||
# E.g. python-mpd2 is actually called python3.6-mpd2
|
||||
# instead of python-3.6-python-mpd2 inside Nixpkgs
|
||||
if req.startswith("python-") or req.startswith("python_"):
|
||||
names.append(req[len("python-") :])
|
||||
if base_req_name.startswith("python-") or base_req_name.startswith("python_"):
|
||||
names.append(base_req_name[len("python-") :])
|
||||
for name in names:
|
||||
# treat "-" and "_" equally
|
||||
name = re.sub("[-_]", "[-_]", name)
|
||||
# python(minor).(major)-(pname)-(version or unstable-date)
|
||||
# we need the version qualifier, or we'll have multiple matches
|
||||
# (e.g. pyserial and pyserial-asyncio when looking for pyserial)
|
||||
pattern = re.compile(
|
||||
f"^python\\d+\\.\\d+-{name}-(?:\\d|unstable-.*)", re.I
|
||||
)
|
||||
pattern = re.compile(rf"^python\d+\.\d+-{name}-(?:[\d\.]+|unstable-.*)", re.I)
|
||||
for attr_path, package in packages.items():
|
||||
# logging.debug("Checking match for %s with %s", name, package["name"])
|
||||
if pattern.match(package["name"]):
|
||||
@@ -130,30 +139,82 @@ def get_cross_provider_reqs(
|
||||
if len(cross_provider_deps[provider]) > 0:
|
||||
reqs.update(
|
||||
chain.from_iterable(
|
||||
get_cross_provider_reqs(
|
||||
d, provider_reqs, cross_provider_deps, seen + [provider]
|
||||
(
|
||||
get_cross_provider_reqs(
|
||||
d, provider_reqs, cross_provider_deps, seen + [provider]
|
||||
)
|
||||
if d not in seen
|
||||
else []
|
||||
)
|
||||
if d not in seen
|
||||
else []
|
||||
for d in cross_provider_deps[provider]
|
||||
)
|
||||
)
|
||||
return reqs
|
||||
|
||||
|
||||
def get_provider_reqs(version: str, packages: Dict) -> Dict:
|
||||
provider_dependencies = get_file_from_github(
|
||||
version, "generated/provider_dependencies.json"
|
||||
)
|
||||
def parse_pyproject_toml(version: str, provider_name: str) -> Dict:
|
||||
provider_dir = provider_name.replace(".", "/")
|
||||
path = f"providers/{provider_dir}/pyproject.toml"
|
||||
try:
|
||||
content = get_file_from_github(version, path)
|
||||
data = tomli.loads(content.decode("utf-8"))
|
||||
|
||||
dependencies = data.get("project", {}).get("dependencies", [])
|
||||
|
||||
# Extract optional dependencies
|
||||
optional_dependencies = data.get("project", {}).get("optional-dependencies", {})
|
||||
for opt_deps_list in optional_dependencies.values():
|
||||
dependencies.extend(opt_deps_list)
|
||||
|
||||
imports = []
|
||||
# Heuristic to generate imports based on provider name
|
||||
# This might not be exhaustive but covers common cases
|
||||
base_import_path = f"airflow.providers.{provider_name.replace('-', '.')}"
|
||||
imports.append(base_import_path)
|
||||
|
||||
# Try to get more specific imports from provider_info entry point if available
|
||||
provider_info_entry_point = (
|
||||
data.get("project", {})
|
||||
.get("entry-points", {})
|
||||
.get("apache_airflow_provider", {})
|
||||
.get("provider_info")
|
||||
)
|
||||
if provider_info_entry_point:
|
||||
module_path = provider_info_entry_point.split(":")[0]
|
||||
if module_path not in imports:
|
||||
imports.append(module_path)
|
||||
|
||||
return {
|
||||
"deps": dependencies,
|
||||
"imports": sorted(list(set(imports))), # Remove duplicates and sort
|
||||
"cross-providers-deps": [], # pyproject.toml doesn't directly list cross-provider deps
|
||||
"version": data.get("project", {}).get("version", "unknown"),
|
||||
}
|
||||
except HTTPError:
|
||||
logging.warning("Couldn't get pyproject.toml for %s", provider_name)
|
||||
return {"deps": [], "imports": [], "cross-providers-deps": []}
|
||||
except Exception as e:
|
||||
logging.error(f"Error parsing pyproject.toml for {provider_name}: {e}")
|
||||
return {"deps": [], "imports": [], "cross-providers-deps": []}
|
||||
|
||||
|
||||
def get_provider_reqs(version: str, packages: Dict, provider_names: List[str]) -> Dict:
|
||||
provider_data = {}
|
||||
for provider in provider_names:
|
||||
data = parse_pyproject_toml(version, provider)
|
||||
provider_data[provider] = {
|
||||
"deps": data["deps"],
|
||||
"cross-providers-deps": data["cross-providers-deps"],
|
||||
}
|
||||
|
||||
provider_reqs = {}
|
||||
cross_provider_deps = {}
|
||||
for provider, provider_data in provider_dependencies.items():
|
||||
for provider, data in provider_data.items():
|
||||
provider_reqs[provider] = list(
|
||||
provider_reqs_to_attr_paths(provider_data["deps"], packages)
|
||||
provider_reqs_to_attr_paths(data["deps"], packages)
|
||||
) + EXTRA_REQS.get(provider, [])
|
||||
cross_provider_deps[provider] = [
|
||||
d for d in provider_data["cross-providers-deps"] if d != "common.sql"
|
||||
]
|
||||
cross_provider_deps[provider] = data["cross-providers-deps"]
|
||||
|
||||
transitive_provider_reqs = {}
|
||||
# Add transitive cross-provider reqs
|
||||
for provider in provider_reqs:
|
||||
@@ -163,39 +224,25 @@ def get_provider_reqs(version: str, packages: Dict) -> Dict:
|
||||
return transitive_provider_reqs
|
||||
|
||||
|
||||
def get_provider_yaml(version: str, provider: str) -> Dict:
|
||||
provider_dir = provider.replace(".", "/")
|
||||
path = f"airflow/providers/{provider_dir}/provider.yaml"
|
||||
try:
|
||||
return get_file_from_github(version, path)
|
||||
except HTTPError:
|
||||
logging.warning("Couldn't get provider yaml for %s", provider)
|
||||
return {}
|
||||
|
||||
|
||||
def get_provider_imports(version: str, providers) -> Dict:
|
||||
def get_provider_imports(version: str, provider_names: List[str]) -> Dict:
|
||||
provider_imports = {}
|
||||
for provider in providers:
|
||||
provider_yaml = get_provider_yaml(version, provider)
|
||||
imports: List[str] = []
|
||||
if "hooks" in provider_yaml:
|
||||
imports.extend(
|
||||
chain.from_iterable(
|
||||
hook["python-modules"] for hook in provider_yaml["hooks"]
|
||||
)
|
||||
)
|
||||
if "operators" in provider_yaml:
|
||||
imports.extend(
|
||||
chain.from_iterable(
|
||||
operator["python-modules"]
|
||||
for operator in provider_yaml["operators"]
|
||||
)
|
||||
)
|
||||
provider_imports[provider] = imports
|
||||
for provider in provider_names:
|
||||
data = parse_pyproject_toml(version, provider)
|
||||
provider_imports[provider] = data["imports"]
|
||||
return provider_imports
|
||||
|
||||
|
||||
def to_nix_expr(provider_reqs: Dict, provider_imports: Dict, fh: TextIO) -> None:
|
||||
def get_provider_versions(version: str, provider_names: List[str]) -> Dict:
|
||||
provider_versions = {}
|
||||
for provider in provider_names:
|
||||
data = parse_pyproject_toml(version, provider)
|
||||
provider_versions[provider] = data["version"]
|
||||
return provider_versions
|
||||
|
||||
|
||||
def to_nix_expr(
|
||||
provider_reqs: Dict, provider_imports: Dict, provider_versions: Dict, fh: TextIO
|
||||
) -> None:
|
||||
fh.write("# Warning: generated by update-providers.py, do not update manually\n")
|
||||
fh.write("{\n")
|
||||
for provider, reqs in provider_reqs.items():
|
||||
@@ -209,19 +256,83 @@ def to_nix_expr(provider_reqs: Dict, provider_imports: Dict, fh: TextIO) -> None
|
||||
+ " ".join(sorted(f'"{imp}"' for imp in provider_imports[provider]))
|
||||
+ " ];\n"
|
||||
)
|
||||
fh.write(f' version = "{provider_versions[provider]}";\n')
|
||||
fh.write(" };\n")
|
||||
fh.write("\n")
|
||||
fh.write("}\n")
|
||||
|
||||
|
||||
def get_all_providers_from_github(version: str) -> List[str]:
|
||||
"""
|
||||
Fetches all provider paths that contain a pyproject.toml file from the airflow repository.
|
||||
This is done by recursively fetching the git tree for the providers directory.
|
||||
"""
|
||||
providers = []
|
||||
try:
|
||||
# Get the SHA of the 'providers' directory for the given version
|
||||
with urlopen(
|
||||
f"https://api.github.com/repos/apache/airflow/contents/?ref={version}"
|
||||
) as response:
|
||||
root_contents = json.loads(response.read())
|
||||
|
||||
providers_dir_info = next(
|
||||
(
|
||||
item
|
||||
for item in root_contents
|
||||
if item["name"] == "providers" and item["type"] == "dir"
|
||||
),
|
||||
None,
|
||||
)
|
||||
|
||||
if not providers_dir_info:
|
||||
logging.error(
|
||||
"Could not find 'providers' directory in the root of the repository."
|
||||
)
|
||||
sys.exit(1)
|
||||
|
||||
providers_dir_sha = providers_dir_info["sha"]
|
||||
|
||||
# Now, get the tree for the 'providers' directory recursively
|
||||
with urlopen(
|
||||
f"https://api.github.com/repos/apache/airflow/git/trees/{providers_dir_sha}?recursive=1"
|
||||
) as response:
|
||||
tree = json.loads(response.read())
|
||||
|
||||
pyproject_paths = [
|
||||
item["path"]
|
||||
for item in tree["tree"]
|
||||
if item["type"] == "blob" and item["path"].endswith("/pyproject.toml")
|
||||
]
|
||||
|
||||
for path in pyproject_paths:
|
||||
# remove /pyproject.toml suffix
|
||||
provider_path = path[: -len("/pyproject.toml")]
|
||||
# Exclude tests directory and empty paths (root pyproject.toml)
|
||||
if provider_path and not provider_path.startswith("tests/"):
|
||||
provider_name = provider_path.replace("/", ".")
|
||||
providers.append(provider_name)
|
||||
|
||||
except HTTPError as e:
|
||||
logging.error(f"Error fetching provider list from GitHub: {e}")
|
||||
sys.exit(1)
|
||||
|
||||
return sorted(providers)
|
||||
|
||||
|
||||
def main() -> None:
|
||||
logging.basicConfig(level=logging.INFO)
|
||||
logging.basicConfig(level=logging.DEBUG)
|
||||
version = get_version()
|
||||
packages = dump_packages()
|
||||
logging.info("Generating providers.nix for version %s", version)
|
||||
provider_reqs = get_provider_reqs(version, packages)
|
||||
provider_imports = get_provider_imports(version, provider_reqs.keys())
|
||||
with open("providers.nix", "w") as fh:
|
||||
to_nix_expr(provider_reqs, provider_imports, fh)
|
||||
|
||||
# Fetch provider names from GitHub API
|
||||
provider_names = get_all_providers_from_github(version)
|
||||
|
||||
provider_reqs = get_provider_reqs(version, packages, provider_names)
|
||||
provider_imports = get_provider_imports(version, provider_names)
|
||||
provider_versions = get_provider_versions(version, provider_names)
|
||||
with open(os.path.join(os.path.dirname(sys.argv[0]), "providers.nix"), "w") as fh:
|
||||
to_nix_expr(provider_reqs, provider_imports, provider_versions, fh)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
Executable
+53
@@ -0,0 +1,53 @@
|
||||
#!/usr/bin/env nix-shell
|
||||
#!nix-shell -i bash -p common-updater-scripts curl jq nixfmt-tree
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
# This script updates apache-airflow to the latest tag for a given major version.
|
||||
# It defaults to major version 3.
|
||||
#
|
||||
# Usage: ./update.sh [MAJOR_VERSION]
|
||||
# Example: ./update.sh 2
|
||||
|
||||
MAJOR_VERSION="${1:-3}"
|
||||
PACKAGE_DIR=$(dirname "$0")
|
||||
PACKAGE_NAME="apache-airflow"
|
||||
|
||||
# Fetches the latest git tag for the given major version from the GitHub API.
|
||||
# It filters for tags that look like version numbers and sorts them to find the latest.
|
||||
get_latest_tag() {
|
||||
TEMP_FILE=$(mktemp)
|
||||
curl -s "https://api.github.com/repos/apache/airflow/releases" > "$TEMP_FILE"
|
||||
LATEST_TAG=$(jq -r --arg major_version "$MAJOR_VERSION" '
|
||||
.[]
|
||||
| select(.tag_name | test("^" + $major_version + "\\.\\d+\\.\\d+$"))
|
||||
| .tag_name
|
||||
' "$TEMP_FILE" | sort -V | tail -n 1)
|
||||
rm "$TEMP_FILE"
|
||||
echo "$LATEST_TAG"
|
||||
}
|
||||
|
||||
echo "Looking for latest tag for major version $MAJOR_VERSION..."
|
||||
LATEST_TAG=$(get_latest_tag)
|
||||
|
||||
if [[ -z "$LATEST_TAG" ]]; then
|
||||
echo "No new tag found for major version $MAJOR_VERSION."
|
||||
exit 0
|
||||
fi
|
||||
|
||||
echo "Latest tag found: $LATEST_TAG"
|
||||
|
||||
# Update the version and hash in the package definition
|
||||
# This uses the standard nixpkgs script `update-source-version`.
|
||||
echo "Updating source version for $PACKAGE_NAME to $LATEST_TAG..."
|
||||
update-source-version "$PACKAGE_NAME" "$LATEST_TAG"
|
||||
|
||||
# After updating the main package, run the providers update script.
|
||||
# It will automatically pick up the new version from the nix file.
|
||||
echo "Updating provider dependencies..."
|
||||
"$PACKAGE_DIR/update-providers.py"
|
||||
echo "Formatting generated providers.nix"
|
||||
treefmt "$PACKAGE_DIR"
|
||||
|
||||
echo "Update complete. Please check the git diff for changes."
|
||||
|
||||
@@ -0,0 +1,76 @@
|
||||
{
|
||||
lib,
|
||||
buildPythonPackage,
|
||||
fetchFromGitHub,
|
||||
hatchling,
|
||||
pytestCheckHook,
|
||||
# runtime dependencies
|
||||
fastapi,
|
||||
issubclass,
|
||||
jinja2,
|
||||
pydantic,
|
||||
starlette,
|
||||
typing-extensions,
|
||||
typing-inspection,
|
||||
# test dependencies
|
||||
dirty-equals,
|
||||
httpx,
|
||||
inline-snapshot,
|
||||
pydantic-settings,
|
||||
pytest-fixture-classes,
|
||||
python-multipart,
|
||||
svcs,
|
||||
typer,
|
||||
uvicorn,
|
||||
pythonAtLeast,
|
||||
}:
|
||||
|
||||
buildPythonPackage rec {
|
||||
pname = "cadwyn";
|
||||
version = "5.6.1";
|
||||
format = "pyproject";
|
||||
|
||||
src = fetchFromGitHub {
|
||||
owner = "zmievsa";
|
||||
repo = "cadwyn";
|
||||
tag = version;
|
||||
hash = "sha256-VVi79c/Y1mu520H/7gy9lGdBIVuKsYedU49P501NQao=";
|
||||
};
|
||||
|
||||
disabled = pythonAtLeast "3.14";
|
||||
|
||||
build-system = [ hatchling ];
|
||||
|
||||
dependencies = [
|
||||
fastapi
|
||||
issubclass
|
||||
jinja2
|
||||
pydantic
|
||||
starlette
|
||||
typing-extensions
|
||||
typing-inspection
|
||||
];
|
||||
|
||||
pythonImportsCheck = [ "cadwyn" ];
|
||||
|
||||
nativeCheckInputs = [
|
||||
dirty-equals
|
||||
httpx
|
||||
inline-snapshot
|
||||
pydantic-settings
|
||||
pytest-fixture-classes
|
||||
pytestCheckHook
|
||||
python-multipart
|
||||
svcs
|
||||
typer
|
||||
uvicorn
|
||||
];
|
||||
|
||||
meta = {
|
||||
description = "Production-ready community-driven modern Stripe-like API versioning in FastAPI";
|
||||
homepage = "https://github.com/zmievsa/cadwyn";
|
||||
changelog = "https://github.com/zmievsa/cadwyn/releases/tag/${version}";
|
||||
license = lib.licenses.mit;
|
||||
maintainers = with lib.maintainers; [ taranarmo ];
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,47 @@
|
||||
{
|
||||
lib,
|
||||
buildPythonPackage,
|
||||
fetchFromGitHub,
|
||||
|
||||
asgiref,
|
||||
greenlet,
|
||||
importlib-metadata,
|
||||
outcome,
|
||||
setuptools,
|
||||
sniffio,
|
||||
}:
|
||||
|
||||
buildPythonPackage rec {
|
||||
pname = "greenback";
|
||||
version = "1.2.1";
|
||||
pyproject = true;
|
||||
|
||||
src = fetchFromGitHub {
|
||||
owner = "oremanj";
|
||||
repo = "greenback";
|
||||
tag = "v${version}";
|
||||
hash = "sha256-u7kihdE5EH3Xbq4dQGY4ojiootjw3muYnoI7d2MrMkE=";
|
||||
};
|
||||
|
||||
build-system = [
|
||||
importlib-metadata
|
||||
setuptools
|
||||
];
|
||||
|
||||
dependencies = [
|
||||
asgiref
|
||||
greenlet
|
||||
outcome
|
||||
sniffio
|
||||
];
|
||||
|
||||
pythonImportsCheck = [ "greenback" ];
|
||||
|
||||
meta = {
|
||||
description = "Use anyio and asyncio from synchronous code";
|
||||
homepage = "https://github.com/oremanj/greenback";
|
||||
changelog = "https://github.com/oremanj/greenback/blob/${src.rev}/CHANGES.rst";
|
||||
license = lib.licenses.mit;
|
||||
maintainers = with lib.maintainers; [ taranarmo ];
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,30 @@
|
||||
{
|
||||
lib,
|
||||
buildPythonPackage,
|
||||
fetchFromGitHub,
|
||||
poetry-core,
|
||||
}:
|
||||
|
||||
buildPythonPackage rec {
|
||||
pname = "issubclass";
|
||||
version = "0.1.2";
|
||||
format = "pyproject";
|
||||
|
||||
src = fetchFromGitHub {
|
||||
owner = "zmievsa";
|
||||
repo = "issubclass";
|
||||
tag = version;
|
||||
hash = "sha256-a4EUbQY5duicnxuwtpBQ/7UBmtmeU+ZaySEnT4lmgLs=";
|
||||
};
|
||||
|
||||
build-system = [ poetry-core ];
|
||||
|
||||
pythonImportsCheck = [ "issubclass" ];
|
||||
|
||||
meta = {
|
||||
description = "issubclass() builtin that doesn't raise TypeError when arguments are not classes";
|
||||
homepage = "https://pypi.org/project/issubclass";
|
||||
license = lib.licenses.mit;
|
||||
maintainers = with lib.maintainers; [ taranarmo ];
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,40 @@
|
||||
{
|
||||
lib,
|
||||
buildPythonPackage,
|
||||
fetchFromGitHub,
|
||||
|
||||
poetry-core,
|
||||
pytest,
|
||||
pytestCheckHook,
|
||||
typing-extensions,
|
||||
}:
|
||||
|
||||
buildPythonPackage rec {
|
||||
pname = "pytest-fixture-classes";
|
||||
version = "1.0.3";
|
||||
format = "pyproject";
|
||||
|
||||
src = fetchFromGitHub {
|
||||
owner = "zmievsa";
|
||||
repo = "pytest-fixture-classes";
|
||||
tag = version;
|
||||
hash = "sha256-A3HsDhCGxoJnkl841tZ/7lIn8UyGy5NRjVWODZV7aOQ=";
|
||||
};
|
||||
|
||||
build-system = [ poetry-core ];
|
||||
|
||||
dependencies = [
|
||||
typing-extensions
|
||||
];
|
||||
|
||||
pythonImportsCheck = [ "pytest_fixture_classes" ];
|
||||
|
||||
nativeCheckInputs = [ pytestCheckHook ];
|
||||
|
||||
meta = {
|
||||
description = "Fixtures as classes that work well with dependency injection, autocompletetion, type checkers, and language servers";
|
||||
homepage = "https://github.com/zmievsa/pytest-fixture-classes";
|
||||
license = lib.licenses.mit;
|
||||
maintainers = with lib.maintainers; [ taranarmo ];
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,50 @@
|
||||
{
|
||||
lib,
|
||||
buildPythonPackage,
|
||||
fetchFromGitHub,
|
||||
# build system
|
||||
setuptools,
|
||||
setuptools-scm,
|
||||
# runtime
|
||||
httpx,
|
||||
pydantic,
|
||||
requests,
|
||||
tenacity,
|
||||
types-requests,
|
||||
}:
|
||||
|
||||
buildPythonPackage rec {
|
||||
pname = "retryhttp";
|
||||
version = "1.3.1";
|
||||
format = "pyproject";
|
||||
|
||||
src = fetchFromGitHub {
|
||||
owner = "austind";
|
||||
repo = "retryhttp";
|
||||
tag = "release/v${version}";
|
||||
hash = "sha256-Jz073CTeIfPjcOQUGdb6/Q5OfEXZW5hTiEPKrfAK0Gg=";
|
||||
};
|
||||
|
||||
build-system = [
|
||||
setuptools
|
||||
setuptools-scm
|
||||
];
|
||||
|
||||
dependencies = [
|
||||
httpx
|
||||
pydantic
|
||||
requests
|
||||
tenacity
|
||||
types-requests
|
||||
];
|
||||
|
||||
pythonImportsCheck = [ "retryhttp" ];
|
||||
|
||||
meta = {
|
||||
description = "Retry potentially transient HTTP errors in Python";
|
||||
homepage = "https://github.com/austind/retryhttp";
|
||||
changelog = "https://github.com/austind/retryhttp/releases/tag/release%2Fv${version}";
|
||||
license = lib.licenses.apsl20;
|
||||
maintainers = with lib.maintainers; [ taranarmo ];
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,73 @@
|
||||
{
|
||||
lib,
|
||||
stdenv,
|
||||
buildPythonPackage,
|
||||
fetchFromGitHub,
|
||||
|
||||
attrs,
|
||||
hatch-fancy-pypi-readme,
|
||||
hatch-vcs,
|
||||
hatchling,
|
||||
# test dependencies
|
||||
aiohttp,
|
||||
fastapi,
|
||||
flask,
|
||||
httpx,
|
||||
pyramid,
|
||||
pytest-asyncio,
|
||||
pytestCheckHook,
|
||||
starlette,
|
||||
sybil,
|
||||
}:
|
||||
|
||||
buildPythonPackage rec {
|
||||
pname = "svcs";
|
||||
version = "25.1.0";
|
||||
format = "pyproject";
|
||||
|
||||
src = fetchFromGitHub {
|
||||
owner = "hynek";
|
||||
repo = "svcs";
|
||||
tag = version;
|
||||
hash = "sha256-dDPmOKGifAGmAH3TD0NzJvR8lUB5qDWbxIwzHtNeF+4=";
|
||||
};
|
||||
|
||||
build-system = [
|
||||
hatch-fancy-pypi-readme
|
||||
hatch-vcs
|
||||
hatchling
|
||||
];
|
||||
|
||||
dependencies = [
|
||||
attrs
|
||||
];
|
||||
|
||||
nativeCheckInputs = [
|
||||
aiohttp
|
||||
fastapi
|
||||
flask
|
||||
httpx
|
||||
pyramid
|
||||
pytest-asyncio
|
||||
pytestCheckHook
|
||||
starlette
|
||||
sybil
|
||||
];
|
||||
|
||||
disabledTests = lib.optionals stdenv.hostPlatform.isDarwin [
|
||||
"test_aclose_registry_ok"
|
||||
"test_registrations"
|
||||
"test_get_pings"
|
||||
"test_client_pool_register_value"
|
||||
];
|
||||
|
||||
pythonImportsCheck = [ "svcs" ];
|
||||
|
||||
meta = {
|
||||
description = "Flexible Service Locator for Python";
|
||||
homepage = "https://github.com/hynek/svcs";
|
||||
changelog = "https://github.com/hynek/svcs/releases/tag/${version}";
|
||||
license = lib.licenses.mit;
|
||||
maintainers = with lib.maintainers; [ taranarmo ];
|
||||
};
|
||||
}
|
||||
@@ -2383,6 +2383,8 @@ self: super: with self; {
|
||||
|
||||
cachy = callPackage ../development/python-modules/cachy { };
|
||||
|
||||
cadwyn = callPackage ../development/python-modules/cadwyn { };
|
||||
|
||||
caffe = toPythonModule (
|
||||
pkgs.caffe.override {
|
||||
pythonSupport = true;
|
||||
@@ -6576,6 +6578,8 @@ self: super: with self; {
|
||||
|
||||
green = callPackage ../development/python-modules/green { };
|
||||
|
||||
greenback = callPackage ../development/python-modules/greenback { };
|
||||
|
||||
greeneye-monitor = callPackage ../development/python-modules/greeneye-monitor { };
|
||||
|
||||
# built-in for pypi
|
||||
@@ -7578,6 +7582,8 @@ self: super: with self; {
|
||||
|
||||
israel-rail-api = callPackage ../development/python-modules/israel-rail-api { };
|
||||
|
||||
issubclass = callPackage ../development/python-modules/issubclass { };
|
||||
|
||||
itables = callPackage ../development/python-modules/itables { };
|
||||
|
||||
itanium-demangler = callPackage ../development/python-modules/itanium-demangler { };
|
||||
@@ -14963,6 +14969,8 @@ self: super: with self; {
|
||||
|
||||
pytest-filter-subpackage = callPackage ../development/python-modules/pytest-filter-subpackage { };
|
||||
|
||||
pytest-fixture-classes = callPackage ../development/python-modules/pytest-fixture-classes { };
|
||||
|
||||
pytest-fixture-config = callPackage ../development/python-modules/pytest-fixture-config { };
|
||||
|
||||
pytest-flake8 = callPackage ../development/python-modules/pytest-flake8 { };
|
||||
@@ -16478,6 +16486,8 @@ self: super: with self; {
|
||||
|
||||
retry2 = callPackage ../development/python-modules/retry2 { };
|
||||
|
||||
retryhttp = callPackage ../development/python-modules/retryhttp { };
|
||||
|
||||
retrying = callPackage ../development/python-modules/retrying { };
|
||||
|
||||
returns = callPackage ../development/python-modules/returns { };
|
||||
@@ -18408,6 +18418,8 @@ self: super: with self; {
|
||||
|
||||
sv-ttk = callPackage ../development/python-modules/sv-ttk { };
|
||||
|
||||
svcs = callPackage ../development/python-modules/svcs { };
|
||||
|
||||
svg-path = callPackage ../development/python-modules/svg-path { };
|
||||
|
||||
svg-py = callPackage ../development/python-modules/svg-py { };
|
||||
|
||||
Reference in New Issue
Block a user