From 9f073cf8d6ca16ed66ca0e3e12f17f6e1b3a8156 Mon Sep 17 00:00:00 2001 From: Phillip Cloud <417981+cpcloud@users.noreply.github.com> Date: Thu, 20 Jan 2022 06:01:45 -0500 Subject: [PATCH 1/4] python3Packages.ibis-framework: 1.3.0 -> 2.1.1 --- .../python-modules/ibis-framework/default.nix | 131 ++++++++++++++---- 1 file changed, 106 insertions(+), 25 deletions(-) diff --git a/pkgs/development/python-modules/ibis-framework/default.nix b/pkgs/development/python-modules/ibis-framework/default.nix index 1627159e6fbd..fa3ba5e99413 100644 --- a/pkgs/development/python-modules/ibis-framework/default.nix +++ b/pkgs/development/python-modules/ibis-framework/default.nix @@ -1,10 +1,22 @@ -{ lib, buildPythonPackage, fetchPypi, fetchpatch, isPy27, pythonAtLeast +{ lib +, buildPythonPackage +, fetchFromGitHub +, pythonOlder +, atpublic +, cached-property +, clickhouse-driver +, click +, dask , graphviz +, importlib-metadata , multipledispatch , numpy , pandas +, parsy , pyarrow , pytest +, pytest-mock +, pytest-xdist , pytz , regex , requests @@ -12,54 +24,123 @@ , tables , toolz }: +let + # ignore tests for which dependencies are not available + backends = [ + "csv" + "dask" + "hdf5" + "pandas" + "parquet" + "sqlite" + ]; + + backendsString = lib.concatStringsSep " " backends; + + ibisTestingData = fetchFromGitHub { + owner = "ibis-project"; + repo = "testing-data"; + rev = "master"; + sha256 = "sha256-xuSE6wHP3aF8lnEE2SuFbTRBu49ecRmc1F3HPcszptI="; + }; +in buildPythonPackage rec { pname = "ibis-framework"; - version = "1.3.0"; - disabled = isPy27 || pythonAtLeast "3.8"; + version = "2.1.1"; + disabled = pythonOlder "3.7"; - src = fetchPypi { - inherit pname version; - sha256 = "1my94a11jzg1hv6ln8wxklbqrg6z5l2l77vr89aq0829yyxacmv7"; + src = fetchFromGitHub { + repo = "ibis"; + owner = "ibis-project"; + rev = "2.1.1"; + sha256 = "sha256-n3fR6wvcSfIo7760seB+5SxtoYSqQmqkzZ9VlNQF200="; }; - patches = [ - # fix tests for pandas 1.1 - (fetchpatch { - url = "https://github.com/ibis-project/ibis/commit/53ef3cefc4ae90d61f3612310cb36da2bcd11305.diff"; - sha256 = "1i5yjmqridjqpggiinsjaz5spcxca5bd48vy7a0mj4mm1b5flw2m"; - }) - ]; - propagatedBuildInputs = [ + atpublic + cached-property + clickhouse-driver + dask + graphviz multipledispatch numpy pandas + parsy + pyarrow pytz regex - toolz - sqlalchemy requests - graphviz + sqlalchemy tables - pyarrow - ]; + toolz + ] ++ lib.optionals (pythonOlder "3.8") [ importlib-metadata ]; checkInputs = [ + click pytest + pytest-mock + pytest-xdist ]; - # ignore tests which require test dataset, or frameworks not available - checkPhase = '' - pytest ibis \ - --ignore=ibis/tests/all \ - --ignore=ibis/{sql,spark} + # these tests are broken upstream: https://github.com/ibis-project/ibis/issues/3291 + disabledTests = [ + "test_summary_numeric" + "test_summary_non_numeric" + "test_batting_most_hits" + "test_join_with_window_function" + "test_where_long" + "test_quantile_groupby" + "test_summary_numeric" + "test_summary_numeric_group_by" + "test_summary_non_numeric" + "test_searched_case_column" + "test_simple_case_column" + "test_summary_non_numeric_group_by" + ]; + + preCheck = '' + set -euo pipefail + + export IBIS_TEST_DATA_DIRECTORY + IBIS_TEST_DATA_DIRECTORY="$(mktemp -d)" + + # copy the test data to a writable directory + cp -r ${ibisTestingData}/* "$IBIS_TEST_DATA_DIRECTORY" + + find "$IBIS_TEST_DATA_DIRECTORY" -type d -exec chmod u+rwx {} + + find "$IBIS_TEST_DATA_DIRECTORY" -type f -exec chmod u+rw {} + + + # load data + for backend in ${backendsString}; do + python ci/datamgr.py "$backend" & + done + + wait + + export PYTEST_BACKENDS="${backendsString}" ''; + checkPhase = '' + set -euo pipefail + + runHook preCheck + + pytest --numprocesses auto \ + ibis/tests \ + ibis/backends/tests \ + ibis/backends/{${lib.concatStringsSep "," backends}}/tests \ + -k '${lib.concatMapStringsSep " and " (test: "not ${test}") disabledTests}' + + runHook postCheck + ''; + + pythonImportsCheck = [ "ibis" ] ++ (map (backend: "ibis.backends.${backend}") backends); + meta = with lib; { description = "Productivity-centric Python Big Data Framework"; homepage = "https://github.com/ibis-project/ibis"; license = licenses.asl20; - maintainers = [ maintainers.costrouc ]; + maintainers = with maintainers; [ costrouc cpcloud ]; }; } From 01665b3dd555f8e16dba7705f32f098d1bb58ce0 Mon Sep 17 00:00:00 2001 From: Phillip Cloud <417981+cpcloud@users.noreply.github.com> Date: Thu, 20 Jan 2022 06:07:21 -0500 Subject: [PATCH 2/4] python3Packages.ibis-framework: use a specific rev for testing-data --- pkgs/development/python-modules/ibis-framework/default.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkgs/development/python-modules/ibis-framework/default.nix b/pkgs/development/python-modules/ibis-framework/default.nix index fa3ba5e99413..838071b4d032 100644 --- a/pkgs/development/python-modules/ibis-framework/default.nix +++ b/pkgs/development/python-modules/ibis-framework/default.nix @@ -40,7 +40,7 @@ let ibisTestingData = fetchFromGitHub { owner = "ibis-project"; repo = "testing-data"; - rev = "master"; + rev = "743201a35c6b968cf55b054f9d28949ea15d1f0a"; sha256 = "sha256-xuSE6wHP3aF8lnEE2SuFbTRBu49ecRmc1F3HPcszptI="; }; in From 5c7f80c45425ad3b7b3b9737bdad476ed16d5173 Mon Sep 17 00:00:00 2001 From: Phillip Cloud <417981+cpcloud@users.noreply.github.com> Date: Thu, 20 Jan 2022 06:07:49 -0500 Subject: [PATCH 3/4] python3Packages.ibis-framework: share the version variable --- pkgs/development/python-modules/ibis-framework/default.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkgs/development/python-modules/ibis-framework/default.nix b/pkgs/development/python-modules/ibis-framework/default.nix index 838071b4d032..707b30da1797 100644 --- a/pkgs/development/python-modules/ibis-framework/default.nix +++ b/pkgs/development/python-modules/ibis-framework/default.nix @@ -53,7 +53,7 @@ buildPythonPackage rec { src = fetchFromGitHub { repo = "ibis"; owner = "ibis-project"; - rev = "2.1.1"; + rev = version; sha256 = "sha256-n3fR6wvcSfIo7760seB+5SxtoYSqQmqkzZ9VlNQF200="; }; From 7965b464ed1bafde3be96c771f78295c3a0ee68e Mon Sep 17 00:00:00 2001 From: Phillip Cloud <417981+cpcloud@users.noreply.github.com> Date: Fri, 21 Jan 2022 16:22:59 -0500 Subject: [PATCH 4/4] python3Packages.ibis-framework: use pytestCheckHook --- .../python-modules/ibis-framework/default.nix | 23 ++++++++----------- 1 file changed, 9 insertions(+), 14 deletions(-) diff --git a/pkgs/development/python-modules/ibis-framework/default.nix b/pkgs/development/python-modules/ibis-framework/default.nix index 707b30da1797..17f4553297ff 100644 --- a/pkgs/development/python-modules/ibis-framework/default.nix +++ b/pkgs/development/python-modules/ibis-framework/default.nix @@ -2,6 +2,7 @@ , buildPythonPackage , fetchFromGitHub , pythonOlder +, pytestCheckHook , atpublic , cached-property , clickhouse-driver @@ -77,6 +78,7 @@ buildPythonPackage rec { ] ++ lib.optionals (pythonOlder "3.8") [ importlib-metadata ]; checkInputs = [ + pytestCheckHook click pytest pytest-mock @@ -99,6 +101,13 @@ buildPythonPackage rec { "test_summary_non_numeric_group_by" ]; + pytestFlagsArray = [ + "--numprocesses $NIX_BUILD_CORES" + "ibis/tests" + "ibis/backends/tests" + "ibis/backends/{${lib.concatStringsSep "," backends}}/tests" + ]; + preCheck = '' set -euo pipefail @@ -121,20 +130,6 @@ buildPythonPackage rec { export PYTEST_BACKENDS="${backendsString}" ''; - checkPhase = '' - set -euo pipefail - - runHook preCheck - - pytest --numprocesses auto \ - ibis/tests \ - ibis/backends/tests \ - ibis/backends/{${lib.concatStringsSep "," backends}}/tests \ - -k '${lib.concatMapStringsSep " and " (test: "not ${test}") disabledTests}' - - runHook postCheck - ''; - pythonImportsCheck = [ "ibis" ] ++ (map (backend: "ibis.backends.${backend}") backends); meta = with lib; {