diff --git a/nixos/tests/prefect.nix b/nixos/tests/prefect.nix index 0e2f1a36c687..c54a69f89e43 100644 --- a/nixos/tests/prefect.nix +++ b/nixos/tests/prefect.nix @@ -11,6 +11,7 @@ in { services.prefect = { enable = true; + baseUrl = "http://127.0.0.1"; }; }; }; @@ -18,7 +19,7 @@ in testScript = '' machine.start() machine.wait_for_unit("prefect-server.service") - machine.wait_for_open_port("${mainPort}") + machine.wait_for_open_port(${mainPort}) ''; meta = with lib.maintainers; { diff --git a/pkgs/by-name/pr/prefect/package.nix b/pkgs/by-name/pr/prefect/package.nix index c23de54e11c9..3665acfe6825 100644 --- a/pkgs/by-name/pr/prefect/package.nix +++ b/pkgs/by-name/pr/prefect/package.nix @@ -1,207 +1 @@ -{ - lib, - python3Packages, - fetchPypi, - nixosTests, - nix-update-script, -}: - -python3Packages.buildPythonApplication rec { - pname = "prefect"; - version = "3.5.0"; - pyproject = true; - - # Trying to install from source is challenging - # the packaging is using versioningit and looking for - # .git directory - # Source will be missing sdist, uv.lock, ui artefacts ... - src = fetchPypi { - inherit pname version; - hash = "sha256-BazzO+gR/mLcx+tlysPMJzcqmctUU0LSYL0YMZMyVrg="; - }; - - pythonRelaxDeps = [ - "websockets" - "typer" - ]; - - build-system = with python3Packages; [ - hatchling - versioningit - ]; - - dependencies = - with python3Packages; - [ - aiosqlite - alembic - apprise - asyncpg - click - cryptography - dateparser - docker - graphviz - jinja2 - jinja2-humanize-extension - humanize - pytz - readchar - sqlalchemy - typer - # client dependencies - anyio - asgi-lifespan - cachetools - cloudpickle - coolname - exceptiongroup - fastapi - fsspec - # graphviz already included - griffe - httpcore - httpx - jsonpatch - jsonschema - opentelemetry-api - orjson - packaging - pathspec - pendulum - pluggy - prometheus-client - pydantic - pydantic-core - pydantic-extra-types - pydantic-settings - python-dateutil - python-slugify - python-socks - pyyaml - rfc3339-validator - rich - ruamel-yaml - semver - sniffio - toml - typing-extensions - ujson - uvicorn - websockets - whenever - uv - semver - ] - ++ sqlalchemy.optional-dependencies.asyncio - ++ httpx.optional-dependencies.http2; - - optional-dependencies = with python3Packages; { - aws = [ - # prefect-aws - ]; - azure = [ - # prefect-azure - ]; - bitbucket = [ - # prefect-bitbucket - ]; - dask = [ - # prefect-dask - ]; - databricks = [ - # prefect-databricks - ]; - dbt = [ - # prefect-dbt - ]; - docker = [ - # prefect-docker - ]; - email = [ - # prefect-email - ]; - gcp = [ - # prefect-gcp - ]; - github = [ - # prefect-github - ]; - gitlab = [ - # prefect-gitlab - ]; - kubernetes = [ - # prefect-kubernetes - ]; - otel = [ - opentelemetry-distro - opentelemetry-exporter-otlp - opentelemetry-instrumentation - opentelemetry-instrumentation-logging - opentelemetry-test-utils - ]; - ray = [ - # prefect-ray - ]; - redis = [ - # prefect-redis - ]; - shell = [ - # prefect-shell - ]; - slack = [ - # prefect-slack - ]; - snowflake = [ - # prefect-snowflake - ]; - sqlalchemy = [ - # prefect-sqlalchemy - ]; - }; - - makeWrapperArgs = [ - # Add the installed directories to the python path so the worker can find them - "--prefix PYTHONPATH : ${python3Packages.makePythonPath dependencies}" - "--prefix PYTHONPATH : $out/${python3Packages.python.sitePackages}" - ]; - - passthru.tests = { - inherit (nixosTests) prefect; - - updateScript = nix-update-script { - extraArgs = [ - # avoid pre‐releases - "--version-regex" - "^(\\d+\\.\\d+\\.\\d+)$" - ]; - }; - }; - - # Tests are not included in the pypi source - doCheck = false; - # nativeCheckInputs = ( - # with python3Packages; - # [ - # pytestCheckHook - # pytest-asyncio - # pytest-cov - # pytest-env - # # pytest-flakefinder - # pytest-mypy-plugins - # pytest-timeout - # pytest-xdist - # ] - # ); - - meta = { - description = "Workflow orchestration framework for building resilient data pipelines in Python"; - homepage = "https://github.com/PrefectHQ/prefect"; - license = lib.licenses.asl20; - maintainers = with lib.maintainers; [ - happysalada - mrmebelman - ]; - mainProgram = "prefect"; - }; -} +{ python3Packages }: python3Packages.toPythonApplication python3Packages.prefect diff --git a/pkgs/development/python-modules/amplitude-analytics/default.nix b/pkgs/development/python-modules/amplitude-analytics/default.nix new file mode 100644 index 000000000000..0f506a7e5322 --- /dev/null +++ b/pkgs/development/python-modules/amplitude-analytics/default.nix @@ -0,0 +1,36 @@ +{ + lib, + buildPythonPackage, + fetchFromGitHub, + pytestCheckHook, + + setuptools, +}: + +buildPythonPackage (finalAttrs: { + pname = "amplitude-analytics"; + version = "1.2.3"; + pyproject = true; + + src = fetchFromGitHub { + owner = "amplitude"; + repo = "Amplitude-Python"; + tag = "v${finalAttrs.version}"; + hash = "sha256-on4TJPiPyznYkBeJsTd7W59KhN7UaSX5+XJaSjkqFaE="; + }; + + build-system = [ setuptools ]; + + nativeCheckInputs = [ pytestCheckHook ]; + + pythonImportsCheck = [ "amplitude" ]; + + meta = { + description = "Official Amplitude backend Python SDK for server-side instrumentation"; + homepage = "https://github.com/amplitude/Amplitude-Python"; + downloadPage = "https://github.com/amplitude/Amplitude-Python/releases"; + changelog = "https://github.com/amplitude/Amplitude-Python/releases/tag/${finalAttrs.src.tag}"; + license = lib.licenses.mit; + maintainers = with lib.maintainers; [ prince213 ]; + }; +}) diff --git a/pkgs/development/python-modules/burner-redis/default.nix b/pkgs/development/python-modules/burner-redis/default.nix new file mode 100644 index 000000000000..c8ffe4e2c8dc --- /dev/null +++ b/pkgs/development/python-modules/burner-redis/default.nix @@ -0,0 +1,52 @@ +{ + lib, + buildPythonPackage, + fetchFromGitHub, + pytestCheckHook, + rustPlatform, + + pytest-asyncio, + redis, +}: + +buildPythonPackage (finalAttrs: { + pname = "burner-redis"; + version = "0.1.6"; + pyproject = true; + + src = fetchFromGitHub { + owner = "prefectlabs"; + repo = "burner-redis"; + tag = "v${finalAttrs.version}"; + hash = "sha256-soXwXxo8vVGEEM5iDNFa86zlPA+e74b+AS9diUoOdME="; + }; + + cargoDeps = rustPlatform.fetchCargoVendor { + inherit (finalAttrs) pname version src; + hash = "sha256-0qdmcoRje7OkHnGhO8CaR/g38yL9K8Jc+1KjfW4xPTQ="; + }; + + nativeBuildInputs = [ + rustPlatform.cargoSetupHook + rustPlatform.maturinBuildHook + ]; + + pythonImportsCheck = [ "burner_redis" ]; + + doCheck = true; + nativeCheckInputs = [ + pytestCheckHook + + pytest-asyncio + redis + ]; + + meta = { + description = "Embedded, in-process Redis-compatible database"; + homepage = "https://github.com/prefectlabs/burner-redis"; + downloadPage = "https://github.com/prefectlabs/burner-redis/releases"; + changelog = "https://github.com/prefectlabs/burner-redis/releases/tag/${finalAttrs.src.tag}"; + license = lib.licenses.mit; + maintainers = with lib.maintainers; [ prince213 ]; + }; +}) diff --git a/pkgs/development/python-modules/fakeredis/default.nix b/pkgs/development/python-modules/fakeredis/default.nix index f2cb17ddb590..74e0f892129b 100644 --- a/pkgs/development/python-modules/fakeredis/default.nix +++ b/pkgs/development/python-modules/fakeredis/default.nix @@ -2,30 +2,32 @@ lib, buildPythonPackage, fetchFromGitHub, + pytestCheckHook, + redisTestHook, + + hatchling, hypothesis, jsonpath-ng, lupa, - hatchling, + numpy, pyprobables, pytest-asyncio, pytest-mock, - pytestCheckHook, redis, - redisTestHook, sortedcontainers, valkey, }: -buildPythonPackage rec { +buildPythonPackage (finalAttrs: { pname = "fakeredis"; - version = "2.33.0"; + version = "2.35.1"; pyproject = true; src = fetchFromGitHub { - owner = "dsoftwareinc"; + owner = "cunla"; repo = "fakeredis-py"; - tag = "v${version}"; - hash = "sha256-uvbvrziVdoa/ip8MbZG8GcpN1FoINxUV+SDVRmg78Qs="; + tag = "v${finalAttrs.version}"; + hash = "sha256-euhWKXFERpRoXX7G81ffAygt5e1mt7uy9Y9zAGacu38="; }; build-system = [ hatchling ]; @@ -42,6 +44,10 @@ buildPythonPackage rec { cf = [ pyprobables ]; probabilistic = [ pyprobables ]; valkey = [ valkey ]; + vectorset = [ + jsonpath-ng + numpy + ]; }; nativeCheckInputs = [ @@ -58,23 +64,8 @@ buildPythonPackage rec { disabledTestMarks = [ "slow" ]; disabledTests = [ - "test_init_args" # AttributeError: module 'fakeredis' has no attribute 'FakeValkey' - "test_async_init_kwargs" # AttributeError: module 'fakeredis' has no attribute 'FakeAsyncValkey'" - # redis.exceptions.ResponseError: unknown command 'evalsha' "test_async_lock" - - # AssertionError: assert [0, b'1'] == [0, 1.0] - "test_zrank_redis7_2" - "test_zrevrank_redis7_2" - - # KeyError: 'tot-mem' - "test_acl_log_auth_exist" - "test_acl_log_invalid_channel" - "test_acl_log_invalid_key" - "test_client_id" - "test_client_info" - "test_client_list" ]; preCheck = '' @@ -83,9 +74,9 @@ buildPythonPackage rec { meta = { description = "Fake implementation of Redis API"; - homepage = "https://github.com/dsoftwareinc/fakeredis-py"; - changelog = "https://github.com/cunla/fakeredis-py/releases/tag/${src.tag}"; + homepage = "https://github.com/cunla/fakeredis-py"; + changelog = "https://github.com/cunla/fakeredis-py/releases/tag/${finalAttrs.src.tag}"; license = with lib.licenses; [ bsd3 ]; maintainers = with lib.maintainers; [ fab ]; }; -} +}) diff --git a/pkgs/development/python-modules/prefect/default.nix b/pkgs/development/python-modules/prefect/default.nix new file mode 100644 index 000000000000..900f3de319e8 --- /dev/null +++ b/pkgs/development/python-modules/prefect/default.nix @@ -0,0 +1,305 @@ +{ + lib, + buildPythonPackage, + fetchFromGitHub, + nixosTests, + nix-update-script, + pytestCheckHook, + pythonAtLeast, + pythonOlder, + replaceVars, + writableTmpDirAsHomeHook, + writeShellScriptBin, + + aiosqlite, + alembic, + amplitude-analytics, + anyio, + apprise, + asgi-lifespan, + asyncpg, + boto3, + cachetools, + click, + cloudpickle, + coolname, + cryptography, + cyclopts, + dateparser, + docker, + exceptiongroup, + fastapi, + fsspec, + graphviz, + griffe, + hatchling, + httpcore, + httpx, + humanize, + jinja2-humanize-extension, + jinja2, + jsonpatch, + jsonschema, + moto, + numpy, + opentelemetry-api, + opentelemetry-distro, + opentelemetry-exporter-otlp, + opentelemetry-instrumentation, + opentelemetry-instrumentation-logging, + opentelemetry-instrumentation-system-metrics, + opentelemetry-sdk, + opentelemetry-test-utils, + orjson, + packaging, + pathspec, + pendulum, + pluggy, + prometheus-client, + pydantic-core, + pydantic-extra-types, + pydantic-settings, + pydantic, + pydocket, + pytest-asyncio, + pytest-env, + pytest-timeout, + pytest-xdist, + python-dateutil, + python-on-whales, + python-slugify, + pytz, + pyyaml, + readchar, + respx, + rfc3339-validator, + rich, + ruamel-yaml-clib, + ruamel-yaml, + semver, + sniffio, + sqlalchemy, + toml, + typing-extensions, + uv, + uvicorn, + versioningit, + watchfiles, + websockets, + whenever, + yamllint, +}: + +buildPythonPackage (finalAttrs: { + pname = "prefect"; + version = "3.7.0"; + pyproject = true; + + src = fetchFromGitHub { + owner = "PrefectHQ"; + repo = "prefect"; + tag = finalAttrs.version; + hash = "sha256-AfiXH9u6W6UpE8hepNzPGIm1cxC+5RonhtBYWMu2IaQ="; + }; + + postPatch = '' + substituteInPlace pyproject.toml \ + --replace-fail \ + 'default-version = "3.6.24+nogit"' \ + 'default-version = "${finalAttrs.version}"' + '' + + lib.optionalString finalAttrs.doCheck '' + substituteInPlace src/prefect/__init__.py \ + --replace-fail \ + '__development_base_path__: pathlib.Path = __module_path__.parents[1]' \ + '__development_base_path__: pathlib.Path = pathlib.Path("${finalAttrs.src}")' + ''; + + # versioningit: NotVCSError: Git not installed; assuming this isn't a Git repository + nativeBuildInputs = [ + (writeShellScriptBin "git" "false") + ]; + + build-system = [ + hatchling + versioningit + ]; + + dependencies = [ + aiosqlite + alembic + apprise + asyncpg + click + cryptography + cyclopts + dateparser + docker + jinja2 + jinja2-humanize-extension + pytz + readchar + sqlalchemy + # client dependencies + amplitude-analytics + anyio + asgi-lifespan + cachetools + cloudpickle + coolname + exceptiongroup + fastapi + fsspec + graphviz + griffe + httpcore + httpx + humanize + jsonpatch + jsonschema + opentelemetry-api + orjson + packaging + pathspec + pluggy + prometheus-client + pydantic + pydantic-core + pydantic-extra-types + pydantic-settings + pydocket + python-dateutil + python-slugify + pyyaml + rfc3339-validator + rich + ruamel-yaml + ruamel-yaml-clib + semver + sniffio + toml + typing-extensions + uvicorn + websockets + ] + ++ sqlalchemy.optional-dependencies.asyncio + ++ httpx.optional-dependencies.http2 + ++ lib.optional (pythonOlder "3.13") pendulum + ++ lib.optional (pythonAtLeast "3.13") whenever; + + optional-dependencies = { + aws = [ + # prefect-aws + ]; + azure = [ + # prefect-azure + ]; + bitbucket = [ + # prefect-bitbucket + ]; + buildx = [ + python-on-whales + ]; + bundles = [ + uv + ]; + dask = [ + # prefect-dask + ]; + databricks = [ + # prefect-databricks + ]; + dbt = [ + # prefect-dbt + ]; + docker = [ + # prefect-docker + ]; + email = [ + # prefect-email + ]; + gcp = [ + # prefect-gcp + ]; + github = [ + # prefect-github + ]; + gitlab = [ + # prefect-gitlab + ]; + kubernetes = [ + # prefect-kubernetes + ]; + otel = [ + opentelemetry-distro + opentelemetry-exporter-otlp + opentelemetry-instrumentation + opentelemetry-instrumentation-logging + opentelemetry-instrumentation-system-metrics + opentelemetry-test-utils + ]; + ray = [ + # prefect-ray + ]; + redis = [ + # prefect-redis + ]; + shell = [ + # prefect-shell + ]; + slack = [ + # prefect-slack + ]; + snowflake = [ + # prefect-snowflake + ]; + sqlalchemy = [ + # prefect-sqlalchemy + ]; + }; + + passthru.tests = { + inherit (nixosTests) prefect; + + updateScript = nix-update-script { + extraArgs = [ + # avoid pre‐releases + "--version-regex" + "^(\\d+\\.\\d+\\.\\d+)$" + ]; + }; + }; + + # FIXME: build killed at ~30% + doCheck = false; + nativeCheckInputs = [ + pytestCheckHook + writableTmpDirAsHomeHook + + boto3 + moto + numpy + opentelemetry-sdk + opentelemetry-test-utils + pytest-asyncio + pytest-env + pytest-timeout + pytest-xdist + respx + uv + uvicorn + watchfiles + yamllint + ]; + + meta = { + description = "Workflow orchestration framework for building resilient data pipelines in Python"; + homepage = "https://github.com/PrefectHQ/prefect"; + license = lib.licenses.asl20; + maintainers = with lib.maintainers; [ + happysalada + mrmebelman + ]; + mainProgram = "prefect"; + }; +}) diff --git a/pkgs/development/python-modules/pydocket/default.nix b/pkgs/development/python-modules/pydocket/default.nix index 0bbd557b70de..256562709b55 100644 --- a/pkgs/development/python-modules/pydocket/default.nix +++ b/pkgs/development/python-modules/pydocket/default.nix @@ -2,17 +2,18 @@ lib, buildPythonPackage, fetchFromGitHub, + pytestCheckHook, # build-system hatch-vcs, hatchling, # dependencies + burner-redis, cloudpickle, - fakeredis, + cronsim, opentelemetry-api, - opentelemetry-exporter-prometheus, - opentelemetry-instrumentation, + opentelemetry-sdk, prometheus-client, py-key-value-aio, python-json-logger, @@ -20,18 +21,25 @@ rich, typer, typing-extensions, + uncalled-for, + + docker, + pytest-asyncio, + pytest-cov, + pytest-timeout, + pytest-xdist, }: buildPythonPackage (finalAttrs: { pname = "pydocket"; - version = "0.17.1"; + version = "0.20.1"; pyproject = true; src = fetchFromGitHub { owner = "chrisguidry"; repo = "docket"; tag = finalAttrs.version; - hash = "sha256-p3FqIHulsBxI7oyiahEruARXBuA2QAYdQeVh83OFHbg="; + hash = "sha256-QCx1tpiMkyIveay3OwnjcTRb8pTJNcTjOyor59oYHqQ="; }; build-system = [ @@ -39,17 +47,11 @@ buildPythonPackage (finalAttrs: { hatchling ]; - pythonRelaxDeps = [ - "fakeredis" - "opentelemetry-exporter-prometheus" - "opentelemetry-instrumentation" - ]; dependencies = [ + burner-redis cloudpickle - fakeredis + cronsim opentelemetry-api - opentelemetry-exporter-prometheus - opentelemetry-instrumentation prometheus-client py-key-value-aio python-json-logger @@ -57,15 +59,31 @@ buildPythonPackage (finalAttrs: { rich typer typing-extensions + uncalled-for ] - ++ fakeredis.optional-dependencies.lua ++ py-key-value-aio.optional-dependencies.memory ++ py-key-value-aio.optional-dependencies.redis; + optional-dependencies = { + metrics = [ + opentelemetry-sdk + ]; + }; + pythonImportsCheck = [ "docket" ]; # All tests require internet access doCheck = false; + nativeCheckInputs = [ + pytestCheckHook + + docker + opentelemetry-sdk + pytest-asyncio + pytest-cov + pytest-timeout + pytest-xdist + ]; meta = { description = "Distributed background task system for Python"; diff --git a/pkgs/development/python-modules/uncalled-for/default.nix b/pkgs/development/python-modules/uncalled-for/default.nix index 7c426ae9ecce..40667a22fd56 100644 --- a/pkgs/development/python-modules/uncalled-for/default.nix +++ b/pkgs/development/python-modules/uncalled-for/default.nix @@ -16,7 +16,7 @@ buildPythonPackage (finalAttrs: { pname = "uncalled-for"; - version = "0.3.1"; + version = "0.3.2"; pyproject = true; __structuredAttrs = true; @@ -25,7 +25,7 @@ buildPythonPackage (finalAttrs: { owner = "chrisguidry"; repo = "uncalled-for"; tag = finalAttrs.version; - hash = "sha256-+akXLsfto3FNbkpsPPwN1DQmvu3BpTafRbqLmLwtqek="; + hash = "sha256-5jFMlynYaxd83SQiZ1uPs1whWFgKP4y6s473TplH4iI="; }; build-system = [ diff --git a/pkgs/top-level/python-packages.nix b/pkgs/top-level/python-packages.nix index adad50c1a32f..ad0a5d278ded 100644 --- a/pkgs/top-level/python-packages.nix +++ b/pkgs/top-level/python-packages.nix @@ -736,6 +736,8 @@ self: super: with self; { ament-package = callPackage ../development/python-modules/ament-package { }; + amplitude-analytics = callPackage ../development/python-modules/amplitude-analytics { }; + amply = callPackage ../development/python-modules/amply { }; amqp = callPackage ../development/python-modules/amqp { }; @@ -2407,6 +2409,8 @@ self: super: with self; { bundlewrap-teamvault = callPackage ../development/python-modules/bundlewrap-teamvault { }; + burner-redis = callPackage ../development/python-modules/burner-redis { }; + busylight-core = callPackage ../development/python-modules/busylight-core { }; busylight-for-humans = callPackage ../development/python-modules/busylight-for-humans { }; @@ -12927,7 +12931,7 @@ self: super: with self; { precisely = callPackage ../development/python-modules/precisely { }; - prefect = toPythonModule pkgs.prefect; + prefect = callPackage ../development/python-modules/prefect { }; prefixed = callPackage ../development/python-modules/prefixed { };