diff --git a/nixos/doc/manual/from_md/release-notes/rl-2111.section.xml b/nixos/doc/manual/from_md/release-notes/rl-2111.section.xml index 03773f964200..86031791b1a4 100644 --- a/nixos/doc/manual/from_md/release-notes/rl-2111.section.xml +++ b/nixos/doc/manual/from_md/release-notes/rl-2111.section.xml @@ -186,6 +186,90 @@
Backward Incompatibilities + + + The paperless module and package have been + removed. All users should migrate to the successor + paperless-ng instead. The Paperless project + has + been archived and advises all users to use + paperless-ng instead. + + + Users can use the services.paperless-ng + module as a replacement while noting the following + incompatibilities: + + + + + services.paperless.ocrLanguages has no + replacement. Users should migrate to + services.paperless-ng.extraConfig + instead: + + + + +{ + services.paperless-ng.extraConfig = { + # Provide languages as ISO 639-2 codes + # separated by a plus (+) sign. + # https://en.wikipedia.org/wiki/List_of_ISO_639-2_codes + PAPERLESS_OCR_LANGUAGE = "deu+eng+jpn"; # German & English & Japanse + }; +} + + + + + If you previously specified + PAPERLESS_CONSUME_MAIL_* settings in + services.paperless.extraConfig you + should remove those options now. You now + must define those settings in the + admin interface of paperless-ng. + + + + + Option services.paperless.manage no + longer exists. Use the script at + ${services.paperless-ng.dataDir}/paperless-ng-manage + instead. Note that this script only exists after the + paperless-ng service has been started + at least once. + + + + + After switching to the new system configuration you should + run the Django management command to reindex your + documents and optionally create a user, if you don’t have + one already. + + + To do so, enter the data directory (the value of + services.paperless-ng.dataDir, + /var/lib/paperless by default), switch + to the paperless user and execute the management command + like below: + + +$ cd /var/lib/paperless +$ su paperless -s /bin/sh +$ ./paperless-ng-manage document_index reindex +# if not already done create a user account, paperless-ng requires a login +$ ./paperless-ng-manage createsuperuser +Username (leave blank to use 'paperless'): my-user-name +Email address: me@example.com +Password: ********** +Password (again): ********** +Superuser created successfully. + + + + The staticjinja package has been upgraded diff --git a/nixos/doc/manual/release-notes/rl-2111.section.md b/nixos/doc/manual/release-notes/rl-2111.section.md index 6d42dfc2dc58..231fc05f88b4 100644 --- a/nixos/doc/manual/release-notes/rl-2111.section.md +++ b/nixos/doc/manual/release-notes/rl-2111.section.md @@ -58,6 +58,53 @@ subsonic-compatible api. Available as [navidrome](#opt-services.navidrome.enable ## Backward Incompatibilities {#sec-release-21.11-incompatibilities} +- The `paperless` module and package have been removed. All users should migrate to the + successor `paperless-ng` instead. The Paperless project [has been + archived](https://github.com/the-paperless-project/paperless/commit/9b0063c9731f7c5f65b1852cb8caff97f5e40ba4) + and advises all users to use `paperless-ng` instead. + + Users can use the `services.paperless-ng` module as a replacement while noting the following incompatibilities: + - `services.paperless.ocrLanguages` has no replacement. Users should migrate to [`services.paperless-ng.extraConfig`](options.html#opt-services.paperless-ng.extraConfig) instead: + ```nix + { + services.paperless-ng.extraConfig = { + # Provide languages as ISO 639-2 codes + # separated by a plus (+) sign. + # https://en.wikipedia.org/wiki/List_of_ISO_639-2_codes + PAPERLESS_OCR_LANGUAGE = "deu+eng+jpn"; # German & English & Japanse + }; + } + ``` + + - If you previously specified `PAPERLESS_CONSUME_MAIL_*` settings in + `services.paperless.extraConfig` you should remove those options now. You + now *must* define those settings in the admin interface of paperless-ng. + + - Option `services.paperless.manage` no longer exists. + Use the script at `${services.paperless-ng.dataDir}/paperless-ng-manage` instead. + Note that this script only exists after the `paperless-ng` service has been + started at least once. + + - After switching to the new system configuration you should run the Django + management command to reindex your documents and optionally create a user, + if you don't have one already. + + To do so, enter the data directory (the value of + `services.paperless-ng.dataDir`, `/var/lib/paperless` by default), switch + to the paperless user and execute the management command like below: + ``` + $ cd /var/lib/paperless + $ su paperless -s /bin/sh + $ ./paperless-ng-manage document_index reindex + # if not already done create a user account, paperless-ng requires a login + $ ./paperless-ng-manage createsuperuser + Username (leave blank to use 'paperless'): my-user-name + Email address: me@example.com + Password: ********** + Password (again): ********** + Superuser created successfully. + ``` + - The `staticjinja` package has been upgraded from 1.0.4 to 3.0.1 - The `erigon` ethereum node has moved to a new database format in `2021-05-04`, and requires a full resync diff --git a/nixos/modules/module-list.nix b/nixos/modules/module-list.nix index 7b638afc2070..f26977b12441 100644 --- a/nixos/modules/module-list.nix +++ b/nixos/modules/module-list.nix @@ -551,7 +551,7 @@ ./services/misc/ombi.nix ./services/misc/osrm.nix ./services/misc/packagekit.nix - ./services/misc/paperless.nix + ./services/misc/paperless-ng.nix ./services/misc/parsoid.nix ./services/misc/plex.nix ./services/misc/plikd.nix diff --git a/nixos/modules/services/misc/paperless-ng.nix b/nixos/modules/services/misc/paperless-ng.nix new file mode 100644 index 000000000000..9eaf8fa8859c --- /dev/null +++ b/nixos/modules/services/misc/paperless-ng.nix @@ -0,0 +1,304 @@ +{ config, pkgs, lib, ... }: + +with lib; +let + cfg = config.services.paperless-ng; + + defaultUser = "paperless"; + + env = { + PAPERLESS_DATA_DIR = cfg.dataDir; + PAPERLESS_MEDIA_ROOT = cfg.mediaDir; + PAPERLESS_CONSUMPTION_DIR = cfg.consumptionDir; + GUNICORN_CMD_ARGS = "--bind=${cfg.address}:${toString cfg.port}"; + } // lib.mapAttrs (_: toString) cfg.extraConfig; + + manage = let + setupEnv = lib.concatStringsSep "\n" (mapAttrsToList (name: val: "export ${name}=\"${val}\"") env); + in pkgs.writeShellScript "manage" '' + ${setupEnv} + exec ${cfg.package}/bin/paperless-ng "$@" + ''; + + # Secure the services + defaultServiceConfig = { + TemporaryFileSystem = "/:ro"; + BindReadOnlyPaths = [ + "/nix/store" + "-/etc/resolv.conf" + "-/etc/nsswitch.conf" + "-/etc/hosts" + "-/etc/localtime" + ]; + BindPaths = [ + cfg.consumptionDir + cfg.dataDir + cfg.mediaDir + ]; + CapabilityBoundingSet = ""; + # ProtectClock adds DeviceAllow=char-rtc r + DeviceAllow = ""; + LockPersonality = true; + MemoryDenyWriteExecute = true; + NoNewPrivileges = true; + PrivateDevices = true; + PrivateMounts = true; + # Needs to connect to redis + # PrivateNetwork = true; + PrivateTmp = true; + PrivateUsers = true; + ProcSubset = "pid"; + ProtectClock = true; + # Breaks if the home dir of the user is in /home + # Also does not add much value in combination with the TemporaryFileSystem. + # ProtectHome = true; + ProtectHostname = true; + # Would re-mount paths ignored by temporary root + #ProtectSystem = "strict"; + ProtectControlGroups = true; + ProtectKernelLogs = true; + ProtectKernelModules = true; + ProtectKernelTunables = true; + ProtectProc = "invisible"; + RestrictAddressFamilies = [ "AF_INET" "AF_INET6" ]; + RestrictNamespaces = true; + RestrictRealtime = true; + RestrictSUIDSGID = true; + SystemCallArchitectures = "native"; + SystemCallFilter = [ "@system-service" "~@privileged @resources @setuid @keyring" ]; + # Does not work well with the temporary root + #UMask = "0066"; + }; +in +{ + meta.maintainers = with maintainers; [ earvstedt Flakebi ]; + + imports = [ + (mkRemovedOptionModule [ "services" "paperless"] '' + The paperless module has been removed as the upstream project died. + Users should migrate to the paperless-ng module (services.paperless-ng). + More information can be found in the NixOS 21.11 release notes. + '') + ]; + + options.services.paperless-ng = { + enable = mkOption { + type = lib.types.bool; + default = false; + description = '' + Enable Paperless-ng. + + When started, the Paperless database is automatically created if it doesn't + exist and updated if the Paperless package has changed. + Both tasks are achieved by running a Django migration. + + A script to manage the Paperless instance (by wrapping Django's manage.py) is linked to + ''${dataDir}/paperless-ng-manage. + ''; + }; + + dataDir = mkOption { + type = types.str; + default = "/var/lib/paperless"; + description = "Directory to store the Paperless data."; + }; + + mediaDir = mkOption { + type = types.str; + default = "${cfg.dataDir}/media"; + defaultText = "\${dataDir}/consume"; + description = "Directory to store the Paperless documents."; + }; + + consumptionDir = mkOption { + type = types.str; + default = "${cfg.dataDir}/consume"; + defaultText = "\${dataDir}/consume"; + description = "Directory from which new documents are imported."; + }; + + consumptionDirIsPublic = mkOption { + type = types.bool; + default = false; + description = "Whether all users can write to the consumption dir."; + }; + + passwordFile = mkOption { + type = types.nullOr types.path; + default = null; + example = "/run/keys/paperless-ng-password"; + description = '' + A file containing the superuser password. + + A superuser is required to access the web interface. + If unset, you can create a superuser manually by running + ''${dataDir}/paperless-ng-manage createsuperuser. + + The default superuser name is admin. To change it, set + option . + WARNING: When changing the superuser name after the initial setup, the old superuser + will continue to exist. + + To disable login for the web interface, set the following: + extraConfig.PAPERLESS_AUTO_LOGIN_USERNAME = "admin";. + WARNING: Only use this on a trusted system without internet access to Paperless. + ''; + }; + + address = mkOption { + type = types.str; + default = "localhost"; + description = "Web interface address."; + }; + + port = mkOption { + type = types.port; + default = 28981; + description = "Web interface port."; + }; + + extraConfig = mkOption { + type = types.attrs; + default = {}; + description = '' + Extra paperless-ng config options. + + See the documentation + for available options. + ''; + example = literalExample '' + { + PAPERLESS_OCR_LANGUAGE = "deu+eng"; + } + ''; + }; + + user = mkOption { + type = types.str; + default = defaultUser; + description = "User under which Paperless runs."; + }; + + package = mkOption { + type = types.package; + default = pkgs.paperless-ng; + defaultText = "pkgs.paperless-ng"; + description = "The Paperless package to use."; + }; + }; + + config = mkIf cfg.enable { + # Enable redis if no special url is set + services.redis.enable = mkIf (!hasAttr "PAPERLESS_REDIS" env) true; + + systemd.tmpfiles.rules = [ + "d '${cfg.dataDir}' - ${cfg.user} ${config.users.users.${cfg.user}.group} - -" + "d '${cfg.mediaDir}' - ${cfg.user} ${config.users.users.${cfg.user}.group} - -" + (if cfg.consumptionDirIsPublic then + "d '${cfg.consumptionDir}' 777 - - - -" + else + "d '${cfg.consumptionDir}' - ${cfg.user} ${config.users.users.${cfg.user}.group} - -" + ) + ]; + + systemd.services.paperless-ng-server = { + description = "Paperless document server"; + serviceConfig = defaultServiceConfig // { + User = cfg.user; + ExecStart = "${cfg.package}/bin/paperless-ng qcluster"; + Restart = "on-failure"; + }; + environment = env; + wantedBy = [ "multi-user.target" ]; + wants = [ "paperless-ng-consumer.service" "paperless-ng-web.service" ]; + + preStart = '' + ln -sf ${manage} ${cfg.dataDir}/paperless-ng-manage + + # Auto-migrate on first run or if the package has changed + versionFile="${cfg.dataDir}/src-version" + if [[ $(cat "$versionFile" 2>/dev/null) != ${cfg.package} ]]; then + ${cfg.package}/bin/paperless-ng migrate + echo ${cfg.package} > "$versionFile" + fi + '' + + optionalString (cfg.passwordFile != null) '' + export PAPERLESS_ADMIN_USER="''${PAPERLESS_ADMIN_USER:-admin}" + export PAPERLESS_ADMIN_PASSWORD=$(cat "${cfg.dataDir}/superuser-password") + superuserState="$PAPERLESS_ADMIN_USER:$PAPERLESS_ADMIN_PASSWORD" + superuserStateFile="${cfg.dataDir}/superuser-state" + + if [[ $(cat "$superuserStateFile" 2>/dev/null) != $superuserState ]]; then + ${cfg.package}/bin/paperless-ng manage_superuser + echo "$superuserState" > "$superuserStateFile" + fi + ''; + }; + + # Password copying can't be implemented as a privileged preStart script + # in 'paperless-ng-server' because 'defaultServiceConfig' limits the filesystem + # paths accessible by the service. + systemd.services.paperless-ng-copy-password = mkIf (cfg.passwordFile != null) { + requiredBy = [ "paperless-ng-server.service" ]; + before = [ "paperless-ng-server.service" ]; + serviceConfig = { + ExecStart = '' + ${pkgs.coreutils}/bin/install --mode 600 --owner '${cfg.user}' --compare \ + '${cfg.passwordFile}' '${cfg.dataDir}/superuser-password' + ''; + Type = "oneshot"; + }; + }; + + systemd.services.paperless-ng-consumer = { + description = "Paperless document consumer"; + serviceConfig = defaultServiceConfig // { + User = cfg.user; + ExecStart = "${cfg.package}/bin/paperless-ng document_consumer"; + Restart = "on-failure"; + }; + environment = env; + # Bind to `paperless-ng-server` so that the consumer never runs + # during migrations + bindsTo = [ "paperless-ng-server.service" ]; + after = [ "paperless-ng-server.service" ]; + }; + + systemd.services.paperless-ng-web = { + description = "Paperless web server"; + serviceConfig = defaultServiceConfig // { + User = cfg.user; + ExecStart = '' + ${pkgs.python3Packages.gunicorn}/bin/gunicorn \ + -c ${cfg.package}/lib/paperless-ng/gunicorn.conf.py paperless.asgi:application + ''; + Restart = "on-failure"; + + AmbientCapabilities = "CAP_NET_BIND_SERVICE"; + CapabilityBoundingSet = "CAP_NET_BIND_SERVICE"; + # gunicorn needs setuid + SystemCallFilter = defaultServiceConfig.SystemCallFilter ++ [ "@setuid" ]; + }; + environment = env // { + PATH = mkForce cfg.package.path; + PYTHONPATH = "${cfg.package.pythonPath}:${cfg.package}/lib/paperless-ng/src"; + }; + # Bind to `paperless-ng-server` so that the web server never runs + # during migrations + bindsTo = [ "paperless-ng-server.service" ]; + after = [ "paperless-ng-server.service" ]; + }; + + users = optionalAttrs (cfg.user == defaultUser) { + users.${defaultUser} = { + group = defaultUser; + uid = config.ids.uids.paperless; + home = cfg.dataDir; + }; + + groups.${defaultUser} = { + gid = config.ids.gids.paperless; + }; + }; + }; +} diff --git a/nixos/modules/services/misc/paperless.nix b/nixos/modules/services/misc/paperless.nix deleted file mode 100644 index 43730b80eb2c..000000000000 --- a/nixos/modules/services/misc/paperless.nix +++ /dev/null @@ -1,183 +0,0 @@ -{ config, pkgs, lib, ... }: - -with lib; -let - cfg = config.services.paperless; - - defaultUser = "paperless"; - - manage = cfg.package.withConfig { - config = { - PAPERLESS_CONSUMPTION_DIR = cfg.consumptionDir; - PAPERLESS_INLINE_DOC = "true"; - PAPERLESS_DISABLE_LOGIN = "true"; - } // cfg.extraConfig; - inherit (cfg) dataDir ocrLanguages; - paperlessPkg = cfg.package; - }; -in -{ - options.services.paperless = { - enable = mkOption { - type = lib.types.bool; - default = false; - description = '' - Enable Paperless. - - When started, the Paperless database is automatically created if it doesn't - exist and updated if the Paperless package has changed. - Both tasks are achieved by running a Django migration. - ''; - }; - - dataDir = mkOption { - type = types.str; - default = "/var/lib/paperless"; - description = "Directory to store the Paperless data."; - }; - - consumptionDir = mkOption { - type = types.str; - default = "${cfg.dataDir}/consume"; - defaultText = "\${dataDir}/consume"; - description = "Directory from which new documents are imported."; - }; - - consumptionDirIsPublic = mkOption { - type = types.bool; - default = false; - description = "Whether all users can write to the consumption dir."; - }; - - ocrLanguages = mkOption { - type = with types; nullOr (listOf str); - default = null; - description = '' - Languages available for OCR via Tesseract, specified as - ISO 639-2/T language codes. - If unset, defaults to all available languages. - ''; - example = [ "eng" "spa" "jpn" ]; - }; - - address = mkOption { - type = types.str; - default = "localhost"; - description = "Server listening address."; - }; - - port = mkOption { - type = types.port; - default = 28981; - description = "Server port to listen on."; - }; - - extraConfig = mkOption { - type = types.attrs; - default = {}; - description = '' - Extra paperless config options. - - The config values are evaluated as double-quoted Bash string literals. - - See paperless-src/paperless.conf.example for available options. - - To enable user authentication, set PAPERLESS_DISABLE_LOGIN = "false" - and run the shell command $dataDir/paperless-manage createsuperuser. - - To define secret options without storing them in /nix/store, use the following pattern: - PAPERLESS_PASSPHRASE = "$(< /etc/my_passphrase_file)" - ''; - example = literalExample '' - { - PAPERLESS_OCR_LANGUAGE = "deu"; - } - ''; - }; - - user = mkOption { - type = types.str; - default = defaultUser; - description = "User under which Paperless runs."; - }; - - package = mkOption { - type = types.package; - default = pkgs.paperless; - defaultText = "pkgs.paperless"; - description = "The Paperless package to use."; - }; - - manage = mkOption { - type = types.package; - readOnly = true; - default = manage; - description = '' - A script to manage the Paperless instance. - It wraps Django's manage.py and is also available at - $dataDir/manage-paperless - ''; - }; - }; - - config = mkIf cfg.enable { - - systemd.tmpfiles.rules = [ - "d '${cfg.dataDir}' - ${cfg.user} ${config.users.users.${cfg.user}.group} - -" - ] ++ (optional cfg.consumptionDirIsPublic - "d '${cfg.consumptionDir}' 777 - - - -" - # If the consumption dir is not created here, it's automatically created by - # 'manage' with the default permissions. - ); - - systemd.services.paperless-consumer = { - description = "Paperless document consumer"; - serviceConfig = { - User = cfg.user; - ExecStart = "${manage} document_consumer"; - Restart = "always"; - }; - after = [ "systemd-tmpfiles-setup.service" ]; - wantedBy = [ "multi-user.target" ]; - preStart = '' - if [[ $(readlink ${cfg.dataDir}/paperless-manage) != ${manage} ]]; then - ln -sf ${manage} ${cfg.dataDir}/paperless-manage - fi - - ${manage.setupEnv} - # Auto-migrate on first run or if the package has changed - versionFile="$PAPERLESS_DBDIR/src-version" - if [[ $(cat "$versionFile" 2>/dev/null) != ${cfg.package} ]]; then - python $paperlessSrc/manage.py migrate - echo ${cfg.package} > "$versionFile" - fi - ''; - }; - - systemd.services.paperless-server = { - description = "Paperless document server"; - serviceConfig = { - User = cfg.user; - ExecStart = "${manage} runserver --noreload ${cfg.address}:${toString cfg.port}"; - Restart = "always"; - }; - # Bind to `paperless-consumer` so that the server never runs - # during migrations - bindsTo = [ "paperless-consumer.service" ]; - after = [ "paperless-consumer.service" ]; - wantedBy = [ "multi-user.target" ]; - }; - - users = optionalAttrs (cfg.user == defaultUser) { - users.${defaultUser} = { - group = defaultUser; - uid = config.ids.uids.paperless; - home = cfg.dataDir; - }; - - groups.${defaultUser} = { - gid = config.ids.gids.paperless; - }; - }; - }; -} diff --git a/nixos/tests/all-tests.nix b/nixos/tests/all-tests.nix index c3dd50007aab..e1ad011b22df 100644 --- a/nixos/tests/all-tests.nix +++ b/nixos/tests/all-tests.nix @@ -336,7 +336,7 @@ in pam-oath-login = handleTest ./pam-oath-login.nix {}; pam-u2f = handleTest ./pam-u2f.nix {}; pantheon = handleTest ./pantheon.nix {}; - paperless = handleTest ./paperless.nix {}; + paperless-ng = handleTest ./paperless-ng.nix {}; pdns-recursor = handleTest ./pdns-recursor.nix {}; peerflix = handleTest ./peerflix.nix {}; pgjwt = handleTest ./pgjwt.nix {}; diff --git a/nixos/tests/paperless-ng.nix b/nixos/tests/paperless-ng.nix new file mode 100644 index 000000000000..d8aafc2a08fd --- /dev/null +++ b/nixos/tests/paperless-ng.nix @@ -0,0 +1,36 @@ +import ./make-test-python.nix ({ lib, ... }: { + name = "paperless-ng"; + meta.maintainers = with lib.maintainers; [ earvstedt Flakebi ]; + + nodes.machine = { pkgs, ... }: { + environment.systemPackages = with pkgs; [ imagemagick jq ]; + services.paperless-ng = { + enable = true; + passwordFile = builtins.toFile "password" "admin"; + }; + virtualisation.memorySize = 1024; + }; + + testScript = '' + machine.wait_for_unit("paperless-ng-consumer.service") + + with subtest("Create test doc"): + machine.succeed( + "convert -size 400x40 xc:white -font 'DejaVu-Sans' -pointsize 20 -fill black " + "-annotate +5+20 'hello world 16-10-2005' /var/lib/paperless/consume/doc.png" + ) + + with subtest("Web interface gets ready"): + machine.wait_for_unit("paperless-ng-web.service") + # Wait until server accepts connections + machine.wait_until_succeeds("curl -fs localhost:28981") + + with subtest("Document is consumed"): + machine.wait_until_succeeds( + "(($(curl -u admin:admin -fs localhost:28981/api/documents/ | jq .count) == 1))" + ) + assert "2005-10-16" in machine.succeed( + "curl -u admin:admin -fs localhost:28981/api/documents/ | jq '.results | .[0] | .created'" + ) + ''; +}) diff --git a/nixos/tests/paperless.nix b/nixos/tests/paperless.nix deleted file mode 100644 index fb83e6f976de..000000000000 --- a/nixos/tests/paperless.nix +++ /dev/null @@ -1,36 +0,0 @@ -import ./make-test-python.nix ({ lib, ... } : { - name = "paperless"; - meta = with lib.maintainers; { - maintainers = [ earvstedt ]; - }; - - machine = { pkgs, ... }: { - environment.systemPackages = with pkgs; [ imagemagick jq ]; - services.paperless = { - enable = true; - ocrLanguages = [ "eng" ]; - }; - }; - - testScript = '' - machine.wait_for_unit("paperless-consumer.service") - - # Create test doc - machine.succeed( - "convert -size 400x40 xc:white -font 'DejaVu-Sans' -pointsize 20 -fill black -annotate +5+20 'hello world 16-10-2005' /var/lib/paperless/consume/doc.png" - ) - - with subtest("Service gets ready"): - machine.wait_for_unit("paperless-server.service") - # Wait until server accepts connections - machine.wait_until_succeeds("curl -fs localhost:28981") - - with subtest("Test document is consumed"): - machine.wait_until_succeeds( - "(($(curl -fs localhost:28981/api/documents/ | jq .count) == 1))" - ) - assert "2005-10-16" in machine.succeed( - "curl -fs localhost:28981/api/documents/ | jq '.results | .[0] | .created'" - ) - ''; -}) diff --git a/pkgs/applications/office/paperless-ng/default.nix b/pkgs/applications/office/paperless-ng/default.nix new file mode 100644 index 000000000000..66a548545b3e --- /dev/null +++ b/pkgs/applications/office/paperless-ng/default.nix @@ -0,0 +1,197 @@ +{ lib +, fetchurl +, nixosTests +, python3 +, ghostscript +, imagemagick +, jbig2enc +, ocrmypdf +, optipng +, pngquant +, qpdf +, tesseract4 +, unpaper +, liberation_ttf +}: + +let + py = python3.override { + packageOverrides = self: super: { + django = super.django_3; + django-picklefield = super.django-picklefield.overrideAttrs (oldAttrs: { + # Checks do not pass with django 3 + doInstallCheck = false; + }); + # Avoid warning in django-q versions > 1.3.4 + # https://github.com/jonaswinkler/paperless-ng/issues/857 + # https://github.com/Koed00/django-q/issues/526 + django-q = super.django-q.overridePythonAttrs (oldAttrs: rec { + version = "1.3.4"; + src = super.fetchPypi { + inherit (oldAttrs) pname; + inherit version; + sha256 = "Uj1U3PG2YVLBtlj5FPAO07UYo0MqnezUiYc4yo274Q8="; + }; + }); + }; + }; + + path = lib.makeBinPath [ ghostscript imagemagick jbig2enc optipng pngquant qpdf tesseract4 unpaper ]; +in +py.pkgs.pythonPackages.buildPythonApplication rec { + pname = "paperless-ng"; + version = "1.4.5"; + + src = fetchurl { + url = "https://github.com/jonaswinkler/paperless-ng/releases/download/ng-${version}/${pname}-${version}.tar.xz"; + sha256 = "2PJb8j3oimlfiJ3gqjK6uTemzFdtAP2Mlm5RH09bx/E="; + }; + + format = "other"; + + # Make bind address configurable + # Fix tests with Pillow 8.3.1: https://github.com/jonaswinkler/paperless-ng/pull/1183 + prePatch = '' + substituteInPlace gunicorn.conf.py --replace "bind = '0.0.0.0:8000'" "" + substituteInPlace src/paperless_tesseract/parsers.py --replace "return x" "return round(x)" + ''; + + propagatedBuildInputs = with py.pkgs.pythonPackages; [ + aioredis + arrow + asgiref + async-timeout + attrs + autobahn + automat + blessed + certifi + cffi + channels-redis + channels + chardet + click + coloredlogs + concurrent-log-handler + constantly + cryptography + daphne + dateparser + django-cors-headers + django_extensions + django-filter + django-picklefield + django-q + django + djangorestframework + filelock + fuzzywuzzy + gunicorn + h11 + hiredis + httptools + humanfriendly + hyperlink + idna + imap-tools + img2pdf + incremental + inotify-simple + inotifyrecursive + joblib + langdetect + lxml + msgpack + numpy + ocrmypdf + pathvalidate + pdfminer + pikepdf + pillow + pluggy + portalocker + psycopg2 + pyasn1-modules + pyasn1 + pycparser + pyopenssl + python-dateutil + python-dotenv + python-gnupg + python-Levenshtein + python_magic + pytz + pyyaml + redis + regex + reportlab + requests + scikit-learn + scipy + service-identity + six + sortedcontainers + sqlparse + threadpoolctl + tika + tqdm + twisted.extras.tls + txaio + tzlocal + urllib3 + uvicorn + uvloop + watchdog + watchgod + wcwidth + websockets + whitenoise + whoosh + zope_interface + ]; + + doCheck = true; + checkInputs = with py.pkgs.pythonPackages; [ + pytest + pytest-cov + pytest-django + pytest-env + pytest-sugar + pytest-xdist + factory_boy + ]; + + # The tests require: + # - PATH with runtime binaries + # - A temporary HOME directory for gnupg + # - XDG_DATA_DIRS with test-specific fonts + checkPhase = '' + pushd src + PATH="${path}:$PATH" HOME=$(mktemp -d) XDG_DATA_DIRS="${liberation_ttf}/share:$XDG_DATA_DIRS" pytest + popd + ''; + + installPhase = '' + mkdir -p $out/lib + cp -r . $out/lib/paperless-ng + chmod +x $out/lib/paperless-ng/src/manage.py + makeWrapper $out/lib/paperless-ng/src/manage.py $out/bin/paperless-ng \ + --prefix PYTHONPATH : "$PYTHONPATH" \ + --prefix PATH : "${path}" + ''; + + passthru = { + # PYTHONPATH of all dependencies used by the package + pythonPath = python3.pkgs.makePythonPath propagatedBuildInputs; + inherit path; + + tests = { inherit (nixosTests) paperless-ng; }; + }; + + meta = with lib; { + description = "A supercharged version of paperless: scan, index, and archive all of your physical documents"; + homepage = "https://paperless-ng.readthedocs.io/en/latest/"; + license = licenses.gpl3Only; + maintainers = with maintainers; [ earvstedt Flakebi ]; + }; +} diff --git a/pkgs/applications/office/paperless/default.nix b/pkgs/applications/office/paperless/default.nix deleted file mode 100644 index 74bdacd95845..000000000000 --- a/pkgs/applications/office/paperless/default.nix +++ /dev/null @@ -1,168 +0,0 @@ -{ stdenv -, lib -, fetchFromGitHub -, makeWrapper -, callPackage - -, python3 -, imagemagick -, ghostscript -, optipng -, tesseract -, unpaper -}: - -## Usage - -# ${paperless}/bin/paperless wraps manage.py - -# ${paperless}/share/paperless/setup-env.sh can be sourced from a -# shell script to setup a Paperless environment - -# paperless.withConfig is a convenience function to setup a -# configured Paperless instance. (See ./withConfig.nix) - -# For WSGI with gunicorn, use a shell script like this: -# let -# pythonEnv = paperless.python.withPackages (ps: paperless.runtimePackages ++ [ ps.gunicorn ]); -# in -# writers.writeBash "run-gunicorn" '' -# source ${paperless}/share/paperless/setup-env.sh -# PYTHONPATH=$paperlessSrc ${pythonEnv}/bin/gunicorn paperless.wsgi -# '' - -let - paperless = stdenv.mkDerivation rec { - pname = "paperless"; - version = "2.7.0"; - - src = fetchFromGitHub { - owner = "the-paperless-project"; - repo = "paperless"; - rev = version; - sha256 = "0pkmyky1crjnsg7r0gfk0fadisfsgzlsq6afpz16wx4hp6yvkkf7"; - }; - - nativeBuildInputs = [ makeWrapper ]; - - doCheck = true; - dontInstall = true; - - pythonEnv = python.withPackages (_: runtimePackages); - pythonCheckEnv = python.withPackages (_: (runtimePackages ++ checkPackages)); - - unpackPhase = '' - srcDir=$out/share/paperless - mkdir -p $srcDir - cp -r --no-preserve=mode $src/src/* $src/LICENSE $srcDir - ''; - - postPatch = '' - # django-cors-headers 3.x requires a scheme for allowed hosts - substituteInPlace $out/share/paperless/paperless/settings.py \ - --replace "localhost:8080" "http://localhost:8080" - ''; - - buildPhase = let - # Paperless has explicit runtime checks that expect these binaries to be in PATH - extraBin = lib.makeBinPath [ imagemagick ghostscript optipng tesseract unpaper ]; - in '' - ${python.interpreter} -m compileall $srcDir - - makeWrapper $pythonEnv/bin/python $out/bin/paperless \ - --set PATH ${extraBin} --add-flags $out/share/paperless/manage.py - - # A shell snippet that can be sourced to setup a paperless env - cat > $out/share/paperless/setup-env.sh <= 3 - factory_boy = customPkgs.factory_boy_2_12_0; - - # These are pre-release versions, hence they are private to this pkg - django-filter = self.callPackage ./python-modules/django-filter.nix {}; - django-crispy-forms = self.callPackage ./python-modules/django-crispy-forms.nix {}; - }; - }; - - runtimePackages = with python.pkgs; [ - dateparser - python-dateutil - django - django-cors-headers - django-crispy-forms - django-filter - django_extensions - djangoql - djangorestframework - factory_boy - filemagic - fuzzywuzzy - langdetect - pdftotext - pillow - psycopg2 - pyocr - python-dotenv - python-gnupg - pytz - termcolor - ] ++ (lib.optional stdenv.isLinux inotify-simple); - - checkPackages = with python.pkgs; [ - pytest - pytest-django - pytest-env - pytest-xdist - ]; - - pyocrWithUserTesseract = pyPkgs: - let - pyocr = pyPkgs.pyocr.override { inherit tesseract; }; - in - if pyocr.outPath == pyPkgs.pyocr.outPath then - pyocr - else - # The user has provided a custom tesseract derivation that might be - # missing some languages that are required for PyOCR's tests. Disable them to - # avoid build errors. - pyocr.overridePythonAttrs (attrs: { - doCheck = false; - }); -in - paperless diff --git a/pkgs/applications/office/paperless/python-modules/default.nix b/pkgs/applications/office/paperless/python-modules/default.nix deleted file mode 100644 index e1fb227614fc..000000000000 --- a/pkgs/applications/office/paperless/python-modules/default.nix +++ /dev/null @@ -1,11 +0,0 @@ -pyPkgs: fetchFromGitHub: -{ - factory_boy_2_12_0 = pyPkgs.factory_boy.overridePythonAttrs (old: rec { - version = "2.12.0"; - src = pyPkgs.fetchPypi { - inherit (old) pname; - inherit version; - sha256 = "0w53hjgag6ad5i2vmrys8ysk54agsqvgbjy9lg8g0d8pi9h8vx7s"; - }; - }); -} diff --git a/pkgs/applications/office/paperless/python-modules/django-crispy-forms.nix b/pkgs/applications/office/paperless/python-modules/django-crispy-forms.nix deleted file mode 100644 index f8b91a94ccc7..000000000000 --- a/pkgs/applications/office/paperless/python-modules/django-crispy-forms.nix +++ /dev/null @@ -1,39 +0,0 @@ -{ lib -, buildPythonPackage -, fetchFromGitHub -, pytestCheckHook -, pytest-django -, django -}: - -buildPythonPackage rec { - pname = "django-crispy-forms"; - version = "1.10.0"; - - src = fetchFromGitHub { - owner = "django-crispy-forms"; - repo = "django-crispy-forms"; - rev = version; - sha256 = "0y6kskfxgckb9npcgwx4zrs5n9px159zh9zhinhxi3i7wlriqpf5"; - }; - - # For reasons unknown, the source dir must contain a dash - # for the tests to run successfully - postUnpack = '' - mv $sourceRoot source- - export sourceRoot=source- - ''; - - checkInputs = [ django pytest-django pytestCheckHook ]; - - preCheck = '' - export DJANGO_SETTINGS_MODULE=crispy_forms.tests.test_settings - ''; - - meta = with lib; { - description = "The best way to have DRY Django forms"; - homepage = "https://github.com/maraujop/django-crispy-forms"; - license = licenses.mit; - maintainers = with maintainers; [ earvstedt ]; - }; -} diff --git a/pkgs/applications/office/paperless/python-modules/django-filter.nix b/pkgs/applications/office/paperless/python-modules/django-filter.nix deleted file mode 100644 index d7f20bd9d37d..000000000000 --- a/pkgs/applications/office/paperless/python-modules/django-filter.nix +++ /dev/null @@ -1,26 +0,0 @@ -{ lib, buildPythonPackage, python, pythonOlder, fetchFromGitHub -, django, django-crispy-forms, djangorestframework, mock, pytz }: - -buildPythonPackage rec { - pname = "django-filter"; - version = "2.1.0-pre"; - disabled = pythonOlder "3.4"; - - src = fetchFromGitHub { - owner = "carltongibson"; - repo = pname; - rev = "24adad8c48bc9e7c7539b6510ffde4ce4effdc29"; - sha256 = "0hv4w95jnlzp9vdximl6bb27fyi75001jhvsbs0ikkd8amq8iaj7"; - }; - - checkInputs = [ django django-crispy-forms djangorestframework mock pytz ]; - - checkPhase = "${python.interpreter} runtests.py"; - - meta = with lib; { - description = "A reusable Django application for allowing users to filter querysets dynamically."; - homepage = "https://github.com/carltongibson/django-filter"; - license = licenses.bsd3; - maintainers = with maintainers; [ earvstedt ]; - }; -} diff --git a/pkgs/applications/office/paperless/withConfig.nix b/pkgs/applications/office/paperless/withConfig.nix deleted file mode 100644 index 652d1478c0ca..000000000000 --- a/pkgs/applications/office/paperless/withConfig.nix +++ /dev/null @@ -1,68 +0,0 @@ -{ paperless, lib, writers }: - -## Usage -# -# nix-build --out-link ./paperless -E ' -# (import {}).paperless.withConfig { -# dataDir = /tmp/paperless-data; -# config = { -# PAPERLESS_DISABLE_LOGIN = "true"; -# }; -# }' -# -# Setup DB -# ./paperless migrate -# -# Consume documents in ${dataDir}/consume -# ./paperless document_consumer --oneshot -# -# Start web interface -# ./paperless runserver --noreload localhost:8000 - -{ config ? {}, dataDir ? null, ocrLanguages ? null -, paperlessPkg ? paperless, extraCmds ? "" }: -with lib; -let - paperless = if ocrLanguages == null then - paperlessPkg - else - (paperlessPkg.override { - tesseract = paperlessPkg.tesseract.override { - enableLanguages = ocrLanguages; - }; - }).overrideDerivation (_: { - # `ocrLanguages` might be missing some languages required by the tests. - doCheck = false; - }); - - envVars = (optionalAttrs (dataDir != null) { - PAPERLESS_CONSUMPTION_DIR = "${dataDir}/consume"; - PAPERLESS_MEDIADIR = "${dataDir}/media"; - PAPERLESS_STATICDIR = "${dataDir}/static"; - PAPERLESS_DBDIR = dataDir; - }) // config; - - envVarDefs = mapAttrsToList (n: v: ''export ${n}="${toString v}"'') envVars; - setupEnvVars = builtins.concatStringsSep "\n" envVarDefs; - - setupEnv = '' - source ${paperless}/share/paperless/setup-env.sh - ${setupEnvVars} - ${optionalString (dataDir != null) '' - mkdir -p "$PAPERLESS_CONSUMPTION_DIR" \ - "$PAPERLESS_MEDIADIR" \ - "$PAPERLESS_STATICDIR" \ - "$PAPERLESS_DBDIR" - ''} - ''; - - runPaperless = writers.writeBash "paperless" '' - set -e - ${setupEnv} - ${extraCmds} - exec python $paperlessSrc/manage.py "$@" - ''; -in - runPaperless // { - inherit paperless setupEnv; - } diff --git a/pkgs/development/python-modules/inotifyrecursive/default.nix b/pkgs/development/python-modules/inotifyrecursive/default.nix new file mode 100644 index 000000000000..d0f81abaa47f --- /dev/null +++ b/pkgs/development/python-modules/inotifyrecursive/default.nix @@ -0,0 +1,28 @@ +{ lib +, buildPythonPackage +, fetchPypi +, inotify-simple +}: + +buildPythonPackage rec { + pname = "inotifyrecursive"; + version = "0.3.5"; + + src = fetchPypi { + inherit pname version; + sha256 = "osRQsxdpPkU4QW+Q6x14WFBtr+a4uIUDe9LdmuLa+h4="; + }; + + propagatedBuildInputs = [ inotify-simple ]; + + # No tests included + doCheck = false; + pythonImportsCheck = [ pname ]; + + meta = with lib; { + description = "Simple recursive inotify watches for Python"; + homepage = "https://github.com/letorbi/inotifyrecursive"; + license = licenses.lgpl3Plus; + maintainers = with maintainers; [ Flakebi ]; + }; +} diff --git a/pkgs/development/python-modules/tika/default.nix b/pkgs/development/python-modules/tika/default.nix new file mode 100644 index 000000000000..375560ad9817 --- /dev/null +++ b/pkgs/development/python-modules/tika/default.nix @@ -0,0 +1,29 @@ +{ lib +, buildPythonPackage +, fetchPypi +, pyyaml +, requests +}: + +buildPythonPackage rec { + pname = "tika"; + version = "1.24"; + + src = fetchPypi { + inherit pname version; + sha256 = "wsUPQFYi90UxhBEE+ehcF1Ea7eEd6OU4XqsaKaMfGRs="; + }; + + propagatedBuildInputs = [ pyyaml requests ]; + + # Requires network + doCheck = false; + pythonImportsCheck = [ pname ]; + + meta = with lib; { + description = "A Python binding to the Apache Tika™ REST services"; + homepage = "https://github.com/chrismattmann/tika-python"; + license = licenses.asl20; + maintainers = with maintainers; [ Flakebi ]; + }; +} diff --git a/pkgs/top-level/aliases.nix b/pkgs/top-level/aliases.nix index eaeff814e13b..2790610e5126 100644 --- a/pkgs/top-level/aliases.nix +++ b/pkgs/top-level/aliases.nix @@ -567,6 +567,7 @@ mapAliases ({ owncloudclient = owncloud-client; # added 2016-08 ocz-ssd-guru = throw "ocz-ssd-guru has been removed due to there being no source available"; # added 2021-07-12 p11_kit = p11-kit; # added 2018-02-25 + paperless = paperless-ng; # added 2021-06-06 parity = openethereum; # added 2020-08-01 parquet-cpp = arrow-cpp; # added 2018-09-08 pass-otp = pass.withExtensions (ext: [ext.pass-otp]); # added 2018-05-04 diff --git a/pkgs/top-level/all-packages.nix b/pkgs/top-level/all-packages.nix index 4ba4c1e4d592..43ecc1288206 100644 --- a/pkgs/top-level/all-packages.nix +++ b/pkgs/top-level/all-packages.nix @@ -7893,7 +7893,7 @@ with pkgs; pamtester = callPackage ../tools/security/pamtester { }; - paperless = callPackage ../applications/office/paperless { }; + paperless-ng = callPackage ../applications/office/paperless-ng { }; paperwork = callPackage ../applications/office/paperwork/paperwork-gtk.nix { }; diff --git a/pkgs/top-level/python-packages.nix b/pkgs/top-level/python-packages.nix index e64b45f8e633..38a487cfcc3f 100644 --- a/pkgs/top-level/python-packages.nix +++ b/pkgs/top-level/python-packages.nix @@ -3585,6 +3585,8 @@ in { inotify-simple = callPackage ../development/python-modules/inotify-simple { }; + inotifyrecursive = callPackage ../development/python-modules/inotifyrecursive { }; + inquirer = callPackage ../development/python-modules/inquirer { }; intake = callPackage ../development/python-modules/intake { }; @@ -8746,6 +8748,8 @@ in { tifffile = callPackage ../development/python-modules/tifffile { }; + tika = callPackage ../development/python-modules/tika { }; + tiledb = callPackage ../development/python-modules/tiledb { inherit (pkgs) tiledb; };