diff --git a/pkgs/by-name/un/unstructured-api/package.nix b/pkgs/by-name/un/unstructured-api/package.nix index adf9fa83dd96..ba94ad44bed4 100644 --- a/pkgs/by-name/un/unstructured-api/package.nix +++ b/pkgs/by-name/un/unstructured-api/package.nix @@ -5,8 +5,6 @@ python3, makeWrapper, nix-update-script, - symlinkJoin, - nltk-data, }: let pythonEnv = python3.withPackages ( @@ -147,14 +145,10 @@ let ++ unstructured.optional-dependencies.all-docs ); version = "0.0.82"; - unstructured_api_nltk_data = symlinkJoin { - name = "unstructured_api_nltk_data"; - - paths = [ - nltk-data.punkt - nltk-data.averaged-perceptron-tagger - ]; - }; + unstructured_api_nltk_data = python3.pkgs.nltk.dataDir (d: [ + d.punkt + d.averaged-perceptron-tagger + ]); in stdenvNoCC.mkDerivation { pname = "unstructured-api"; diff --git a/pkgs/development/python-modules/aider-chat/default.nix b/pkgs/development/python-modules/aider-chat/default.nix index bc4534b8da61..3d961c34a4fd 100644 --- a/pkgs/development/python-modules/aider-chat/default.nix +++ b/pkgs/development/python-modules/aider-chat/default.nix @@ -6,8 +6,6 @@ gitMinimal, portaudio, playwright-driver, - symlinkJoin, - nltk-data, pythonOlder, pythonAtLeast, setuptools-scm, @@ -122,13 +120,10 @@ }: let - aider-nltk-data = symlinkJoin { - name = "aider-nltk-data"; - paths = [ - nltk-data.punkt-tab - nltk-data.stopwords - ]; - }; + aider-nltk-data = nltk.dataDir (d: [ + d.punkt-tab + d.stopwords + ]); version = "0.83.1"; aider-chat = buildPythonPackage { diff --git a/pkgs/development/python-modules/nltk/data-dir.nix b/pkgs/development/python-modules/nltk/data-dir.nix new file mode 100644 index 000000000000..d18ab730597b --- /dev/null +++ b/pkgs/development/python-modules/nltk/data-dir.nix @@ -0,0 +1,15 @@ +{ + lib, + pkgs, + python3Packages, +}: +lib.makeOverridable ( + { ... }@nltkDataPkgs: + f: + pkgs.symlinkJoin { + inherit (python3Packages.nltk) meta; + name = "nltk-data-dir"; + + paths = f nltkDataPkgs; + } +) python3Packages.nltk.data diff --git a/pkgs/development/python-modules/nltk/default.nix b/pkgs/development/python-modules/nltk/default.nix index 4493348bb6da..4571f85bf23a 100644 --- a/pkgs/development/python-modules/nltk/default.nix +++ b/pkgs/development/python-modules/nltk/default.nix @@ -1,5 +1,6 @@ { lib, + pkgs, fetchPypi, buildPythonPackage, pythonOlder, @@ -7,6 +8,16 @@ joblib, regex, tqdm, + + # preInstallCheck + nltk, + + # nativeCheckInputs + matplotlib, + numpy, + pyparsing, + pytestCheckHook, + pytest-mock, }: buildPythonPackage rec { @@ -21,28 +32,78 @@ buildPythonPackage rec { hash = "sha256-h9EnvT3kvYmk+BJl5fpZyxsZmydEAXU3D3QX0rx66Gg="; }; - propagatedBuildInputs = [ + dependencies = [ click joblib regex tqdm ]; - # Tests require some data, the downloading of which is impure. It would - # probably make sense to make the data another derivation, but then feeding - # that into the tests (given that we need nltk itself to download the data, - # unless there's an easy way to download it without nltk's downloader) might - # be complicated. For now let's just disable the tests and hope for the - # best. - doCheck = false; + # Use new passthru function to pass dependencies required for testing + preInstallCheck = '' + export NLTK_DATA=${ + nltk.dataDir ( + d: with d; [ + averaged-perceptron-tagger-eng + averaged-perceptron-tagger-rus + brown + cess-cat + cess-esp + conll2007 + floresta + gutenberg + inaugural + indian + large-grammars + nombank-1-0 + omw-1-4 + pl196x + porter-test + ptb + punkt-tab + rte + sinica-treebank + stopwords + tagsets-json + treebank + twitter-samples + udhr + universal-tagset + wmt15-eval + wordnet + wordnet-ic + words + ] + ) + } + ''; + + nativeCheckInputs = [ + pytestCheckHook + matplotlib + numpy + pyparsing + pytest-mock + + pkgs.which + ]; + + disabledTestPaths = [ + "nltk/test/unit/test_downloader.py" # Touches network + ]; pythonImportsCheck = [ "nltk" ]; + passthru = { + data = pkgs.nltk-data; + dataDir = pkgs.callPackage ./data-dir.nix { }; + }; + meta = with lib; { description = "Natural Language Processing ToolKit"; mainProgram = "nltk"; homepage = "http://nltk.org/"; license = licenses.asl20; - maintainers = [ ]; + maintainers = [ lib.maintainers.bengsparks ]; }; } diff --git a/pkgs/development/python-modules/type-infer/default.nix b/pkgs/development/python-modules/type-infer/default.nix index 4b8ef9e99d5e..371708f7d0d0 100644 --- a/pkgs/development/python-modules/type-infer/default.nix +++ b/pkgs/development/python-modules/type-infer/default.nix @@ -16,18 +16,13 @@ python-dateutil, scipy, toml, - nltk-data, - symlinkJoin, }: let - testNltkData = symlinkJoin { - name = "nltk-test-data"; - paths = [ - nltk-data.punkt - nltk-data.punkt-tab - nltk-data.stopwords - ]; - }; + testNltkData = nltk.dataDir (d: [ + d.punkt + d.punkt-tab + d.stopwords + ]); version = "0.0.21"; tag = "v${version}";