From c9caba6f7d6d0d768246ff72f877733be5f35a60 Mon Sep 17 00:00:00 2001 From: Benjamin Sparks Date: Wed, 21 May 2025 13:39:51 +0200 Subject: [PATCH 1/8] python3Packages.nltk: add bengsparks to maintainers --- pkgs/development/python-modules/nltk/default.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkgs/development/python-modules/nltk/default.nix b/pkgs/development/python-modules/nltk/default.nix index 4493348bb6da..30496070153b 100644 --- a/pkgs/development/python-modules/nltk/default.nix +++ b/pkgs/development/python-modules/nltk/default.nix @@ -43,6 +43,6 @@ buildPythonPackage rec { mainProgram = "nltk"; homepage = "http://nltk.org/"; license = licenses.asl20; - maintainers = [ ]; + maintainers = [ lib.maintainers.bengsparks ]; }; } From 9562488efb1b8e2d60e60203e9ea752c131cad2c Mon Sep 17 00:00:00 2001 From: Benjamin Sparks Date: Wed, 21 May 2025 15:51:49 +0200 Subject: [PATCH 2/8] python3Packages.nltk.data: access `nltk-data` packages via `passthru` --- pkgs/development/python-modules/nltk/default.nix | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/pkgs/development/python-modules/nltk/default.nix b/pkgs/development/python-modules/nltk/default.nix index 30496070153b..55831f7708ae 100644 --- a/pkgs/development/python-modules/nltk/default.nix +++ b/pkgs/development/python-modules/nltk/default.nix @@ -1,5 +1,6 @@ { lib, + pkgs, fetchPypi, buildPythonPackage, pythonOlder, @@ -38,6 +39,10 @@ buildPythonPackage rec { pythonImportsCheck = [ "nltk" ]; + passthru = { + data = pkgs.nltk-data; + }; + meta = with lib; { description = "Natural Language Processing ToolKit"; mainProgram = "nltk"; From aee430ee47e1e80e4d555722e3d6ba1bf7899c00 Mon Sep 17 00:00:00 2001 From: Benjamin Sparks Date: Wed, 21 May 2025 16:02:39 +0200 Subject: [PATCH 3/8] python3Packages.nltk.dataDir: init Add helper function to install NLTK data packages like `wordnet` and `punkt-tab`. Should be used in conjunction with `makeWrapperArgs`, e.g. `makeWrapperArgs = [ "--set" "NLTK_DATA" "${nltk.dataDir(d: [ d.wordnet d.punkt-tab ])}" ];` --- pkgs/development/python-modules/nltk/data-dir.nix | 15 +++++++++++++++ pkgs/development/python-modules/nltk/default.nix | 1 + 2 files changed, 16 insertions(+) create mode 100644 pkgs/development/python-modules/nltk/data-dir.nix diff --git a/pkgs/development/python-modules/nltk/data-dir.nix b/pkgs/development/python-modules/nltk/data-dir.nix new file mode 100644 index 000000000000..d18ab730597b --- /dev/null +++ b/pkgs/development/python-modules/nltk/data-dir.nix @@ -0,0 +1,15 @@ +{ + lib, + pkgs, + python3Packages, +}: +lib.makeOverridable ( + { ... }@nltkDataPkgs: + f: + pkgs.symlinkJoin { + inherit (python3Packages.nltk) meta; + name = "nltk-data-dir"; + + paths = f nltkDataPkgs; + } +) python3Packages.nltk.data diff --git a/pkgs/development/python-modules/nltk/default.nix b/pkgs/development/python-modules/nltk/default.nix index 55831f7708ae..bdb79081ec81 100644 --- a/pkgs/development/python-modules/nltk/default.nix +++ b/pkgs/development/python-modules/nltk/default.nix @@ -41,6 +41,7 @@ buildPythonPackage rec { passthru = { data = pkgs.nltk-data; + dataDir = pkgs.callPackage ./data-dir.nix { }; }; meta = with lib; { From acc6352eeb3e1b4fb10f6d17954bc0eb4edf07f7 Mon Sep 17 00:00:00 2001 From: Benjamin Sparks Date: Wed, 21 May 2025 16:32:59 +0200 Subject: [PATCH 4/8] python3Packages.nltk: propagatedBuildInputs -> dependencies --- pkgs/development/python-modules/nltk/default.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkgs/development/python-modules/nltk/default.nix b/pkgs/development/python-modules/nltk/default.nix index bdb79081ec81..be7a461af3ef 100644 --- a/pkgs/development/python-modules/nltk/default.nix +++ b/pkgs/development/python-modules/nltk/default.nix @@ -22,7 +22,7 @@ buildPythonPackage rec { hash = "sha256-h9EnvT3kvYmk+BJl5fpZyxsZmydEAXU3D3QX0rx66Gg="; }; - propagatedBuildInputs = [ + dependencies = [ click joblib regex From 279f2c0308682c2177595cc3af6d1e26fc084160 Mon Sep 17 00:00:00 2001 From: Benjamin Sparks Date: Sat, 24 May 2025 18:26:51 +0200 Subject: [PATCH 5/8] python3Packages.nltk: run tests --- .../python-modules/nltk/default.nix | 69 +++++++++++++++++-- 1 file changed, 62 insertions(+), 7 deletions(-) diff --git a/pkgs/development/python-modules/nltk/default.nix b/pkgs/development/python-modules/nltk/default.nix index be7a461af3ef..4571f85bf23a 100644 --- a/pkgs/development/python-modules/nltk/default.nix +++ b/pkgs/development/python-modules/nltk/default.nix @@ -8,6 +8,16 @@ joblib, regex, tqdm, + + # preInstallCheck + nltk, + + # nativeCheckInputs + matplotlib, + numpy, + pyparsing, + pytestCheckHook, + pytest-mock, }: buildPythonPackage rec { @@ -29,13 +39,58 @@ buildPythonPackage rec { tqdm ]; - # Tests require some data, the downloading of which is impure. It would - # probably make sense to make the data another derivation, but then feeding - # that into the tests (given that we need nltk itself to download the data, - # unless there's an easy way to download it without nltk's downloader) might - # be complicated. For now let's just disable the tests and hope for the - # best. - doCheck = false; + # Use new passthru function to pass dependencies required for testing + preInstallCheck = '' + export NLTK_DATA=${ + nltk.dataDir ( + d: with d; [ + averaged-perceptron-tagger-eng + averaged-perceptron-tagger-rus + brown + cess-cat + cess-esp + conll2007 + floresta + gutenberg + inaugural + indian + large-grammars + nombank-1-0 + omw-1-4 + pl196x + porter-test + ptb + punkt-tab + rte + sinica-treebank + stopwords + tagsets-json + treebank + twitter-samples + udhr + universal-tagset + wmt15-eval + wordnet + wordnet-ic + words + ] + ) + } + ''; + + nativeCheckInputs = [ + pytestCheckHook + matplotlib + numpy + pyparsing + pytest-mock + + pkgs.which + ]; + + disabledTestPaths = [ + "nltk/test/unit/test_downloader.py" # Touches network + ]; pythonImportsCheck = [ "nltk" ]; From 0bb5be9addc86bff92d3ede45ec094755055cba5 Mon Sep 17 00:00:00 2001 From: Benjamin Sparks Date: Sat, 24 May 2025 18:51:58 +0200 Subject: [PATCH 6/8] unstructured-api: pkgs.symlinkJoin -> python.pkgs.nltk.dataDir --- pkgs/by-name/un/unstructured-api/package.nix | 14 ++++---------- 1 file changed, 4 insertions(+), 10 deletions(-) diff --git a/pkgs/by-name/un/unstructured-api/package.nix b/pkgs/by-name/un/unstructured-api/package.nix index adf9fa83dd96..ba94ad44bed4 100644 --- a/pkgs/by-name/un/unstructured-api/package.nix +++ b/pkgs/by-name/un/unstructured-api/package.nix @@ -5,8 +5,6 @@ python3, makeWrapper, nix-update-script, - symlinkJoin, - nltk-data, }: let pythonEnv = python3.withPackages ( @@ -147,14 +145,10 @@ let ++ unstructured.optional-dependencies.all-docs ); version = "0.0.82"; - unstructured_api_nltk_data = symlinkJoin { - name = "unstructured_api_nltk_data"; - - paths = [ - nltk-data.punkt - nltk-data.averaged-perceptron-tagger - ]; - }; + unstructured_api_nltk_data = python3.pkgs.nltk.dataDir (d: [ + d.punkt + d.averaged-perceptron-tagger + ]); in stdenvNoCC.mkDerivation { pname = "unstructured-api"; From 8aa49dac246c0ff3657823556fd6b52534cf6aed Mon Sep 17 00:00:00 2001 From: Benjamin Sparks Date: Sat, 24 May 2025 18:53:56 +0200 Subject: [PATCH 7/8] python3Packages.aider-chat: pkgs.symlinkJoin -> python3Packages.nltk.dataDir --- .../python-modules/aider-chat/default.nix | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/pkgs/development/python-modules/aider-chat/default.nix b/pkgs/development/python-modules/aider-chat/default.nix index bc4534b8da61..3d961c34a4fd 100644 --- a/pkgs/development/python-modules/aider-chat/default.nix +++ b/pkgs/development/python-modules/aider-chat/default.nix @@ -6,8 +6,6 @@ gitMinimal, portaudio, playwright-driver, - symlinkJoin, - nltk-data, pythonOlder, pythonAtLeast, setuptools-scm, @@ -122,13 +120,10 @@ }: let - aider-nltk-data = symlinkJoin { - name = "aider-nltk-data"; - paths = [ - nltk-data.punkt-tab - nltk-data.stopwords - ]; - }; + aider-nltk-data = nltk.dataDir (d: [ + d.punkt-tab + d.stopwords + ]); version = "0.83.1"; aider-chat = buildPythonPackage { From 020c1daaf9062c0066f43646bd8b86b562b412a7 Mon Sep 17 00:00:00 2001 From: Benjamin Sparks Date: Sat, 24 May 2025 18:56:48 +0200 Subject: [PATCH 8/8] python3Packages.type-infer: pkgs.symlinkJoin -> python3Packages.nltk.dataDir --- .../python-modules/type-infer/default.nix | 15 +++++---------- 1 file changed, 5 insertions(+), 10 deletions(-) diff --git a/pkgs/development/python-modules/type-infer/default.nix b/pkgs/development/python-modules/type-infer/default.nix index 4b8ef9e99d5e..371708f7d0d0 100644 --- a/pkgs/development/python-modules/type-infer/default.nix +++ b/pkgs/development/python-modules/type-infer/default.nix @@ -16,18 +16,13 @@ python-dateutil, scipy, toml, - nltk-data, - symlinkJoin, }: let - testNltkData = symlinkJoin { - name = "nltk-test-data"; - paths = [ - nltk-data.punkt - nltk-data.punkt-tab - nltk-data.stopwords - ]; - }; + testNltkData = nltk.dataDir (d: [ + d.punkt + d.punkt-tab + d.stopwords + ]); version = "0.0.21"; tag = "v${version}";