Merge pull request #123046 from Flakebi/paperless

paperless-ng: init at 1.4.5
This commit is contained in:
Andreas Rammhold
2021-08-18 14:39:03 +02:00
committed by GitHub
19 changed files with 733 additions and 534 deletions
@@ -186,6 +186,90 @@
<section xml:id="sec-release-21.11-incompatibilities">
<title>Backward Incompatibilities</title>
<itemizedlist>
<listitem>
<para>
The <literal>paperless</literal> module and package have been
removed. All users should migrate to the successor
<literal>paperless-ng</literal> instead. The Paperless project
<link xlink:href="https://github.com/the-paperless-project/paperless/commit/9b0063c9731f7c5f65b1852cb8caff97f5e40ba4">has
been archived</link> and advises all users to use
<literal>paperless-ng</literal> instead.
</para>
<para>
Users can use the <literal>services.paperless-ng</literal>
module as a replacement while noting the following
incompatibilities:
</para>
<itemizedlist spacing="compact">
<listitem>
<para>
<literal>services.paperless.ocrLanguages</literal> has no
replacement. Users should migrate to
<link xlink:href="options.html#opt-services.paperless-ng.extraConfig"><literal>services.paperless-ng.extraConfig</literal></link>
instead:
</para>
</listitem>
</itemizedlist>
<programlisting language="bash">
{
services.paperless-ng.extraConfig = {
# Provide languages as ISO 639-2 codes
# separated by a plus (+) sign.
# https://en.wikipedia.org/wiki/List_of_ISO_639-2_codes
PAPERLESS_OCR_LANGUAGE = &quot;deu+eng+jpn&quot;; # German &amp; English &amp; Japanse
};
}
</programlisting>
<itemizedlist>
<listitem>
<para>
If you previously specified
<literal>PAPERLESS_CONSUME_MAIL_*</literal> settings in
<literal>services.paperless.extraConfig</literal> you
should remove those options now. You now
<emphasis>must</emphasis> define those settings in the
admin interface of paperless-ng.
</para>
</listitem>
<listitem>
<para>
Option <literal>services.paperless.manage</literal> no
longer exists. Use the script at
<literal>${services.paperless-ng.dataDir}/paperless-ng-manage</literal>
instead. Note that this script only exists after the
<literal>paperless-ng</literal> service has been started
at least once.
</para>
</listitem>
<listitem>
<para>
After switching to the new system configuration you should
run the Django management command to reindex your
documents and optionally create a user, if you dont have
one already.
</para>
<para>
To do so, enter the data directory (the value of
<literal>services.paperless-ng.dataDir</literal>,
<literal>/var/lib/paperless</literal> by default), switch
to the paperless user and execute the management command
like below:
</para>
<programlisting>
$ cd /var/lib/paperless
$ su paperless -s /bin/sh
$ ./paperless-ng-manage document_index reindex
# if not already done create a user account, paperless-ng requires a login
$ ./paperless-ng-manage createsuperuser
Username (leave blank to use 'paperless'): my-user-name
Email address: me@example.com
Password: **********
Password (again): **********
Superuser created successfully.
</programlisting>
</listitem>
</itemizedlist>
</listitem>
<listitem>
<para>
The <literal>staticjinja</literal> package has been upgraded
@@ -58,6 +58,53 @@ subsonic-compatible api. Available as [navidrome](#opt-services.navidrome.enable
## Backward Incompatibilities {#sec-release-21.11-incompatibilities}
- The `paperless` module and package have been removed. All users should migrate to the
successor `paperless-ng` instead. The Paperless project [has been
archived](https://github.com/the-paperless-project/paperless/commit/9b0063c9731f7c5f65b1852cb8caff97f5e40ba4)
and advises all users to use `paperless-ng` instead.
Users can use the `services.paperless-ng` module as a replacement while noting the following incompatibilities:
- `services.paperless.ocrLanguages` has no replacement. Users should migrate to [`services.paperless-ng.extraConfig`](options.html#opt-services.paperless-ng.extraConfig) instead:
```nix
{
services.paperless-ng.extraConfig = {
# Provide languages as ISO 639-2 codes
# separated by a plus (+) sign.
# https://en.wikipedia.org/wiki/List_of_ISO_639-2_codes
PAPERLESS_OCR_LANGUAGE = "deu+eng+jpn"; # German & English & Japanse
};
}
```
- If you previously specified `PAPERLESS_CONSUME_MAIL_*` settings in
`services.paperless.extraConfig` you should remove those options now. You
now *must* define those settings in the admin interface of paperless-ng.
- Option `services.paperless.manage` no longer exists.
Use the script at `${services.paperless-ng.dataDir}/paperless-ng-manage` instead.
Note that this script only exists after the `paperless-ng` service has been
started at least once.
- After switching to the new system configuration you should run the Django
management command to reindex your documents and optionally create a user,
if you don't have one already.
To do so, enter the data directory (the value of
`services.paperless-ng.dataDir`, `/var/lib/paperless` by default), switch
to the paperless user and execute the management command like below:
```
$ cd /var/lib/paperless
$ su paperless -s /bin/sh
$ ./paperless-ng-manage document_index reindex
# if not already done create a user account, paperless-ng requires a login
$ ./paperless-ng-manage createsuperuser
Username (leave blank to use 'paperless'): my-user-name
Email address: me@example.com
Password: **********
Password (again): **********
Superuser created successfully.
```
- The `staticjinja` package has been upgraded from 1.0.4 to 3.0.1
- The `erigon` ethereum node has moved to a new database format in `2021-05-04`, and requires a full resync
+1 -1
View File
@@ -551,7 +551,7 @@
./services/misc/ombi.nix
./services/misc/osrm.nix
./services/misc/packagekit.nix
./services/misc/paperless.nix
./services/misc/paperless-ng.nix
./services/misc/parsoid.nix
./services/misc/plex.nix
./services/misc/plikd.nix
@@ -0,0 +1,304 @@
{ config, pkgs, lib, ... }:
with lib;
let
cfg = config.services.paperless-ng;
defaultUser = "paperless";
env = {
PAPERLESS_DATA_DIR = cfg.dataDir;
PAPERLESS_MEDIA_ROOT = cfg.mediaDir;
PAPERLESS_CONSUMPTION_DIR = cfg.consumptionDir;
GUNICORN_CMD_ARGS = "--bind=${cfg.address}:${toString cfg.port}";
} // lib.mapAttrs (_: toString) cfg.extraConfig;
manage = let
setupEnv = lib.concatStringsSep "\n" (mapAttrsToList (name: val: "export ${name}=\"${val}\"") env);
in pkgs.writeShellScript "manage" ''
${setupEnv}
exec ${cfg.package}/bin/paperless-ng "$@"
'';
# Secure the services
defaultServiceConfig = {
TemporaryFileSystem = "/:ro";
BindReadOnlyPaths = [
"/nix/store"
"-/etc/resolv.conf"
"-/etc/nsswitch.conf"
"-/etc/hosts"
"-/etc/localtime"
];
BindPaths = [
cfg.consumptionDir
cfg.dataDir
cfg.mediaDir
];
CapabilityBoundingSet = "";
# ProtectClock adds DeviceAllow=char-rtc r
DeviceAllow = "";
LockPersonality = true;
MemoryDenyWriteExecute = true;
NoNewPrivileges = true;
PrivateDevices = true;
PrivateMounts = true;
# Needs to connect to redis
# PrivateNetwork = true;
PrivateTmp = true;
PrivateUsers = true;
ProcSubset = "pid";
ProtectClock = true;
# Breaks if the home dir of the user is in /home
# Also does not add much value in combination with the TemporaryFileSystem.
# ProtectHome = true;
ProtectHostname = true;
# Would re-mount paths ignored by temporary root
#ProtectSystem = "strict";
ProtectControlGroups = true;
ProtectKernelLogs = true;
ProtectKernelModules = true;
ProtectKernelTunables = true;
ProtectProc = "invisible";
RestrictAddressFamilies = [ "AF_INET" "AF_INET6" ];
RestrictNamespaces = true;
RestrictRealtime = true;
RestrictSUIDSGID = true;
SystemCallArchitectures = "native";
SystemCallFilter = [ "@system-service" "~@privileged @resources @setuid @keyring" ];
# Does not work well with the temporary root
#UMask = "0066";
};
in
{
meta.maintainers = with maintainers; [ earvstedt Flakebi ];
imports = [
(mkRemovedOptionModule [ "services" "paperless"] ''
The paperless module has been removed as the upstream project died.
Users should migrate to the paperless-ng module (services.paperless-ng).
More information can be found in the NixOS 21.11 release notes.
'')
];
options.services.paperless-ng = {
enable = mkOption {
type = lib.types.bool;
default = false;
description = ''
Enable Paperless-ng.
When started, the Paperless database is automatically created if it doesn't
exist and updated if the Paperless package has changed.
Both tasks are achieved by running a Django migration.
A script to manage the Paperless instance (by wrapping Django's manage.py) is linked to
<literal>''${dataDir}/paperless-ng-manage</literal>.
'';
};
dataDir = mkOption {
type = types.str;
default = "/var/lib/paperless";
description = "Directory to store the Paperless data.";
};
mediaDir = mkOption {
type = types.str;
default = "${cfg.dataDir}/media";
defaultText = "\${dataDir}/consume";
description = "Directory to store the Paperless documents.";
};
consumptionDir = mkOption {
type = types.str;
default = "${cfg.dataDir}/consume";
defaultText = "\${dataDir}/consume";
description = "Directory from which new documents are imported.";
};
consumptionDirIsPublic = mkOption {
type = types.bool;
default = false;
description = "Whether all users can write to the consumption dir.";
};
passwordFile = mkOption {
type = types.nullOr types.path;
default = null;
example = "/run/keys/paperless-ng-password";
description = ''
A file containing the superuser password.
A superuser is required to access the web interface.
If unset, you can create a superuser manually by running
<literal>''${dataDir}/paperless-ng-manage createsuperuser</literal>.
The default superuser name is <literal>admin</literal>. To change it, set
option <option>extraConfig.PAPERLESS_ADMIN_USER</option>.
WARNING: When changing the superuser name after the initial setup, the old superuser
will continue to exist.
To disable login for the web interface, set the following:
<literal>extraConfig.PAPERLESS_AUTO_LOGIN_USERNAME = "admin";</literal>.
WARNING: Only use this on a trusted system without internet access to Paperless.
'';
};
address = mkOption {
type = types.str;
default = "localhost";
description = "Web interface address.";
};
port = mkOption {
type = types.port;
default = 28981;
description = "Web interface port.";
};
extraConfig = mkOption {
type = types.attrs;
default = {};
description = ''
Extra paperless-ng config options.
See <link xlink:href="https://paperless-ng.readthedocs.io/en/latest/configuration.html">the documentation</link>
for available options.
'';
example = literalExample ''
{
PAPERLESS_OCR_LANGUAGE = "deu+eng";
}
'';
};
user = mkOption {
type = types.str;
default = defaultUser;
description = "User under which Paperless runs.";
};
package = mkOption {
type = types.package;
default = pkgs.paperless-ng;
defaultText = "pkgs.paperless-ng";
description = "The Paperless package to use.";
};
};
config = mkIf cfg.enable {
# Enable redis if no special url is set
services.redis.enable = mkIf (!hasAttr "PAPERLESS_REDIS" env) true;
systemd.tmpfiles.rules = [
"d '${cfg.dataDir}' - ${cfg.user} ${config.users.users.${cfg.user}.group} - -"
"d '${cfg.mediaDir}' - ${cfg.user} ${config.users.users.${cfg.user}.group} - -"
(if cfg.consumptionDirIsPublic then
"d '${cfg.consumptionDir}' 777 - - - -"
else
"d '${cfg.consumptionDir}' - ${cfg.user} ${config.users.users.${cfg.user}.group} - -"
)
];
systemd.services.paperless-ng-server = {
description = "Paperless document server";
serviceConfig = defaultServiceConfig // {
User = cfg.user;
ExecStart = "${cfg.package}/bin/paperless-ng qcluster";
Restart = "on-failure";
};
environment = env;
wantedBy = [ "multi-user.target" ];
wants = [ "paperless-ng-consumer.service" "paperless-ng-web.service" ];
preStart = ''
ln -sf ${manage} ${cfg.dataDir}/paperless-ng-manage
# Auto-migrate on first run or if the package has changed
versionFile="${cfg.dataDir}/src-version"
if [[ $(cat "$versionFile" 2>/dev/null) != ${cfg.package} ]]; then
${cfg.package}/bin/paperless-ng migrate
echo ${cfg.package} > "$versionFile"
fi
''
+ optionalString (cfg.passwordFile != null) ''
export PAPERLESS_ADMIN_USER="''${PAPERLESS_ADMIN_USER:-admin}"
export PAPERLESS_ADMIN_PASSWORD=$(cat "${cfg.dataDir}/superuser-password")
superuserState="$PAPERLESS_ADMIN_USER:$PAPERLESS_ADMIN_PASSWORD"
superuserStateFile="${cfg.dataDir}/superuser-state"
if [[ $(cat "$superuserStateFile" 2>/dev/null) != $superuserState ]]; then
${cfg.package}/bin/paperless-ng manage_superuser
echo "$superuserState" > "$superuserStateFile"
fi
'';
};
# Password copying can't be implemented as a privileged preStart script
# in 'paperless-ng-server' because 'defaultServiceConfig' limits the filesystem
# paths accessible by the service.
systemd.services.paperless-ng-copy-password = mkIf (cfg.passwordFile != null) {
requiredBy = [ "paperless-ng-server.service" ];
before = [ "paperless-ng-server.service" ];
serviceConfig = {
ExecStart = ''
${pkgs.coreutils}/bin/install --mode 600 --owner '${cfg.user}' --compare \
'${cfg.passwordFile}' '${cfg.dataDir}/superuser-password'
'';
Type = "oneshot";
};
};
systemd.services.paperless-ng-consumer = {
description = "Paperless document consumer";
serviceConfig = defaultServiceConfig // {
User = cfg.user;
ExecStart = "${cfg.package}/bin/paperless-ng document_consumer";
Restart = "on-failure";
};
environment = env;
# Bind to `paperless-ng-server` so that the consumer never runs
# during migrations
bindsTo = [ "paperless-ng-server.service" ];
after = [ "paperless-ng-server.service" ];
};
systemd.services.paperless-ng-web = {
description = "Paperless web server";
serviceConfig = defaultServiceConfig // {
User = cfg.user;
ExecStart = ''
${pkgs.python3Packages.gunicorn}/bin/gunicorn \
-c ${cfg.package}/lib/paperless-ng/gunicorn.conf.py paperless.asgi:application
'';
Restart = "on-failure";
AmbientCapabilities = "CAP_NET_BIND_SERVICE";
CapabilityBoundingSet = "CAP_NET_BIND_SERVICE";
# gunicorn needs setuid
SystemCallFilter = defaultServiceConfig.SystemCallFilter ++ [ "@setuid" ];
};
environment = env // {
PATH = mkForce cfg.package.path;
PYTHONPATH = "${cfg.package.pythonPath}:${cfg.package}/lib/paperless-ng/src";
};
# Bind to `paperless-ng-server` so that the web server never runs
# during migrations
bindsTo = [ "paperless-ng-server.service" ];
after = [ "paperless-ng-server.service" ];
};
users = optionalAttrs (cfg.user == defaultUser) {
users.${defaultUser} = {
group = defaultUser;
uid = config.ids.uids.paperless;
home = cfg.dataDir;
};
groups.${defaultUser} = {
gid = config.ids.gids.paperless;
};
};
};
}
-183
View File
@@ -1,183 +0,0 @@
{ config, pkgs, lib, ... }:
with lib;
let
cfg = config.services.paperless;
defaultUser = "paperless";
manage = cfg.package.withConfig {
config = {
PAPERLESS_CONSUMPTION_DIR = cfg.consumptionDir;
PAPERLESS_INLINE_DOC = "true";
PAPERLESS_DISABLE_LOGIN = "true";
} // cfg.extraConfig;
inherit (cfg) dataDir ocrLanguages;
paperlessPkg = cfg.package;
};
in
{
options.services.paperless = {
enable = mkOption {
type = lib.types.bool;
default = false;
description = ''
Enable Paperless.
When started, the Paperless database is automatically created if it doesn't
exist and updated if the Paperless package has changed.
Both tasks are achieved by running a Django migration.
'';
};
dataDir = mkOption {
type = types.str;
default = "/var/lib/paperless";
description = "Directory to store the Paperless data.";
};
consumptionDir = mkOption {
type = types.str;
default = "${cfg.dataDir}/consume";
defaultText = "\${dataDir}/consume";
description = "Directory from which new documents are imported.";
};
consumptionDirIsPublic = mkOption {
type = types.bool;
default = false;
description = "Whether all users can write to the consumption dir.";
};
ocrLanguages = mkOption {
type = with types; nullOr (listOf str);
default = null;
description = ''
Languages available for OCR via Tesseract, specified as
<literal>ISO 639-2/T</literal> language codes.
If unset, defaults to all available languages.
'';
example = [ "eng" "spa" "jpn" ];
};
address = mkOption {
type = types.str;
default = "localhost";
description = "Server listening address.";
};
port = mkOption {
type = types.port;
default = 28981;
description = "Server port to listen on.";
};
extraConfig = mkOption {
type = types.attrs;
default = {};
description = ''
Extra paperless config options.
The config values are evaluated as double-quoted Bash string literals.
See <literal>paperless-src/paperless.conf.example</literal> for available options.
To enable user authentication, set <literal>PAPERLESS_DISABLE_LOGIN = "false"</literal>
and run the shell command <literal>$dataDir/paperless-manage createsuperuser</literal>.
To define secret options without storing them in /nix/store, use the following pattern:
<literal>PAPERLESS_PASSPHRASE = "$(&lt; /etc/my_passphrase_file)"</literal>
'';
example = literalExample ''
{
PAPERLESS_OCR_LANGUAGE = "deu";
}
'';
};
user = mkOption {
type = types.str;
default = defaultUser;
description = "User under which Paperless runs.";
};
package = mkOption {
type = types.package;
default = pkgs.paperless;
defaultText = "pkgs.paperless";
description = "The Paperless package to use.";
};
manage = mkOption {
type = types.package;
readOnly = true;
default = manage;
description = ''
A script to manage the Paperless instance.
It wraps Django's manage.py and is also available at
<literal>$dataDir/manage-paperless</literal>
'';
};
};
config = mkIf cfg.enable {
systemd.tmpfiles.rules = [
"d '${cfg.dataDir}' - ${cfg.user} ${config.users.users.${cfg.user}.group} - -"
] ++ (optional cfg.consumptionDirIsPublic
"d '${cfg.consumptionDir}' 777 - - - -"
# If the consumption dir is not created here, it's automatically created by
# 'manage' with the default permissions.
);
systemd.services.paperless-consumer = {
description = "Paperless document consumer";
serviceConfig = {
User = cfg.user;
ExecStart = "${manage} document_consumer";
Restart = "always";
};
after = [ "systemd-tmpfiles-setup.service" ];
wantedBy = [ "multi-user.target" ];
preStart = ''
if [[ $(readlink ${cfg.dataDir}/paperless-manage) != ${manage} ]]; then
ln -sf ${manage} ${cfg.dataDir}/paperless-manage
fi
${manage.setupEnv}
# Auto-migrate on first run or if the package has changed
versionFile="$PAPERLESS_DBDIR/src-version"
if [[ $(cat "$versionFile" 2>/dev/null) != ${cfg.package} ]]; then
python $paperlessSrc/manage.py migrate
echo ${cfg.package} > "$versionFile"
fi
'';
};
systemd.services.paperless-server = {
description = "Paperless document server";
serviceConfig = {
User = cfg.user;
ExecStart = "${manage} runserver --noreload ${cfg.address}:${toString cfg.port}";
Restart = "always";
};
# Bind to `paperless-consumer` so that the server never runs
# during migrations
bindsTo = [ "paperless-consumer.service" ];
after = [ "paperless-consumer.service" ];
wantedBy = [ "multi-user.target" ];
};
users = optionalAttrs (cfg.user == defaultUser) {
users.${defaultUser} = {
group = defaultUser;
uid = config.ids.uids.paperless;
home = cfg.dataDir;
};
groups.${defaultUser} = {
gid = config.ids.gids.paperless;
};
};
};
}
+1 -1
View File
@@ -336,7 +336,7 @@ in
pam-oath-login = handleTest ./pam-oath-login.nix {};
pam-u2f = handleTest ./pam-u2f.nix {};
pantheon = handleTest ./pantheon.nix {};
paperless = handleTest ./paperless.nix {};
paperless-ng = handleTest ./paperless-ng.nix {};
pdns-recursor = handleTest ./pdns-recursor.nix {};
peerflix = handleTest ./peerflix.nix {};
pgjwt = handleTest ./pgjwt.nix {};
+36
View File
@@ -0,0 +1,36 @@
import ./make-test-python.nix ({ lib, ... }: {
name = "paperless-ng";
meta.maintainers = with lib.maintainers; [ earvstedt Flakebi ];
nodes.machine = { pkgs, ... }: {
environment.systemPackages = with pkgs; [ imagemagick jq ];
services.paperless-ng = {
enable = true;
passwordFile = builtins.toFile "password" "admin";
};
virtualisation.memorySize = 1024;
};
testScript = ''
machine.wait_for_unit("paperless-ng-consumer.service")
with subtest("Create test doc"):
machine.succeed(
"convert -size 400x40 xc:white -font 'DejaVu-Sans' -pointsize 20 -fill black "
"-annotate +5+20 'hello world 16-10-2005' /var/lib/paperless/consume/doc.png"
)
with subtest("Web interface gets ready"):
machine.wait_for_unit("paperless-ng-web.service")
# Wait until server accepts connections
machine.wait_until_succeeds("curl -fs localhost:28981")
with subtest("Document is consumed"):
machine.wait_until_succeeds(
"(($(curl -u admin:admin -fs localhost:28981/api/documents/ | jq .count) == 1))"
)
assert "2005-10-16" in machine.succeed(
"curl -u admin:admin -fs localhost:28981/api/documents/ | jq '.results | .[0] | .created'"
)
'';
})
-36
View File
@@ -1,36 +0,0 @@
import ./make-test-python.nix ({ lib, ... } : {
name = "paperless";
meta = with lib.maintainers; {
maintainers = [ earvstedt ];
};
machine = { pkgs, ... }: {
environment.systemPackages = with pkgs; [ imagemagick jq ];
services.paperless = {
enable = true;
ocrLanguages = [ "eng" ];
};
};
testScript = ''
machine.wait_for_unit("paperless-consumer.service")
# Create test doc
machine.succeed(
"convert -size 400x40 xc:white -font 'DejaVu-Sans' -pointsize 20 -fill black -annotate +5+20 'hello world 16-10-2005' /var/lib/paperless/consume/doc.png"
)
with subtest("Service gets ready"):
machine.wait_for_unit("paperless-server.service")
# Wait until server accepts connections
machine.wait_until_succeeds("curl -fs localhost:28981")
with subtest("Test document is consumed"):
machine.wait_until_succeeds(
"(($(curl -fs localhost:28981/api/documents/ | jq .count) == 1))"
)
assert "2005-10-16" in machine.succeed(
"curl -fs localhost:28981/api/documents/ | jq '.results | .[0] | .created'"
)
'';
})
@@ -0,0 +1,197 @@
{ lib
, fetchurl
, nixosTests
, python3
, ghostscript
, imagemagick
, jbig2enc
, ocrmypdf
, optipng
, pngquant
, qpdf
, tesseract4
, unpaper
, liberation_ttf
}:
let
py = python3.override {
packageOverrides = self: super: {
django = super.django_3;
django-picklefield = super.django-picklefield.overrideAttrs (oldAttrs: {
# Checks do not pass with django 3
doInstallCheck = false;
});
# Avoid warning in django-q versions > 1.3.4
# https://github.com/jonaswinkler/paperless-ng/issues/857
# https://github.com/Koed00/django-q/issues/526
django-q = super.django-q.overridePythonAttrs (oldAttrs: rec {
version = "1.3.4";
src = super.fetchPypi {
inherit (oldAttrs) pname;
inherit version;
sha256 = "Uj1U3PG2YVLBtlj5FPAO07UYo0MqnezUiYc4yo274Q8=";
};
});
};
};
path = lib.makeBinPath [ ghostscript imagemagick jbig2enc optipng pngquant qpdf tesseract4 unpaper ];
in
py.pkgs.pythonPackages.buildPythonApplication rec {
pname = "paperless-ng";
version = "1.4.5";
src = fetchurl {
url = "https://github.com/jonaswinkler/paperless-ng/releases/download/ng-${version}/${pname}-${version}.tar.xz";
sha256 = "2PJb8j3oimlfiJ3gqjK6uTemzFdtAP2Mlm5RH09bx/E=";
};
format = "other";
# Make bind address configurable
# Fix tests with Pillow 8.3.1: https://github.com/jonaswinkler/paperless-ng/pull/1183
prePatch = ''
substituteInPlace gunicorn.conf.py --replace "bind = '0.0.0.0:8000'" ""
substituteInPlace src/paperless_tesseract/parsers.py --replace "return x" "return round(x)"
'';
propagatedBuildInputs = with py.pkgs.pythonPackages; [
aioredis
arrow
asgiref
async-timeout
attrs
autobahn
automat
blessed
certifi
cffi
channels-redis
channels
chardet
click
coloredlogs
concurrent-log-handler
constantly
cryptography
daphne
dateparser
django-cors-headers
django_extensions
django-filter
django-picklefield
django-q
django
djangorestframework
filelock
fuzzywuzzy
gunicorn
h11
hiredis
httptools
humanfriendly
hyperlink
idna
imap-tools
img2pdf
incremental
inotify-simple
inotifyrecursive
joblib
langdetect
lxml
msgpack
numpy
ocrmypdf
pathvalidate
pdfminer
pikepdf
pillow
pluggy
portalocker
psycopg2
pyasn1-modules
pyasn1
pycparser
pyopenssl
python-dateutil
python-dotenv
python-gnupg
python-Levenshtein
python_magic
pytz
pyyaml
redis
regex
reportlab
requests
scikit-learn
scipy
service-identity
six
sortedcontainers
sqlparse
threadpoolctl
tika
tqdm
twisted.extras.tls
txaio
tzlocal
urllib3
uvicorn
uvloop
watchdog
watchgod
wcwidth
websockets
whitenoise
whoosh
zope_interface
];
doCheck = true;
checkInputs = with py.pkgs.pythonPackages; [
pytest
pytest-cov
pytest-django
pytest-env
pytest-sugar
pytest-xdist
factory_boy
];
# The tests require:
# - PATH with runtime binaries
# - A temporary HOME directory for gnupg
# - XDG_DATA_DIRS with test-specific fonts
checkPhase = ''
pushd src
PATH="${path}:$PATH" HOME=$(mktemp -d) XDG_DATA_DIRS="${liberation_ttf}/share:$XDG_DATA_DIRS" pytest
popd
'';
installPhase = ''
mkdir -p $out/lib
cp -r . $out/lib/paperless-ng
chmod +x $out/lib/paperless-ng/src/manage.py
makeWrapper $out/lib/paperless-ng/src/manage.py $out/bin/paperless-ng \
--prefix PYTHONPATH : "$PYTHONPATH" \
--prefix PATH : "${path}"
'';
passthru = {
# PYTHONPATH of all dependencies used by the package
pythonPath = python3.pkgs.makePythonPath propagatedBuildInputs;
inherit path;
tests = { inherit (nixosTests) paperless-ng; };
};
meta = with lib; {
description = "A supercharged version of paperless: scan, index, and archive all of your physical documents";
homepage = "https://paperless-ng.readthedocs.io/en/latest/";
license = licenses.gpl3Only;
maintainers = with maintainers; [ earvstedt Flakebi ];
};
}
@@ -1,168 +0,0 @@
{ stdenv
, lib
, fetchFromGitHub
, makeWrapper
, callPackage
, python3
, imagemagick
, ghostscript
, optipng
, tesseract
, unpaper
}:
## Usage
# ${paperless}/bin/paperless wraps manage.py
# ${paperless}/share/paperless/setup-env.sh can be sourced from a
# shell script to setup a Paperless environment
# paperless.withConfig is a convenience function to setup a
# configured Paperless instance. (See ./withConfig.nix)
# For WSGI with gunicorn, use a shell script like this:
# let
# pythonEnv = paperless.python.withPackages (ps: paperless.runtimePackages ++ [ ps.gunicorn ]);
# in
# writers.writeBash "run-gunicorn" ''
# source ${paperless}/share/paperless/setup-env.sh
# PYTHONPATH=$paperlessSrc ${pythonEnv}/bin/gunicorn paperless.wsgi
# ''
let
paperless = stdenv.mkDerivation rec {
pname = "paperless";
version = "2.7.0";
src = fetchFromGitHub {
owner = "the-paperless-project";
repo = "paperless";
rev = version;
sha256 = "0pkmyky1crjnsg7r0gfk0fadisfsgzlsq6afpz16wx4hp6yvkkf7";
};
nativeBuildInputs = [ makeWrapper ];
doCheck = true;
dontInstall = true;
pythonEnv = python.withPackages (_: runtimePackages);
pythonCheckEnv = python.withPackages (_: (runtimePackages ++ checkPackages));
unpackPhase = ''
srcDir=$out/share/paperless
mkdir -p $srcDir
cp -r --no-preserve=mode $src/src/* $src/LICENSE $srcDir
'';
postPatch = ''
# django-cors-headers 3.x requires a scheme for allowed hosts
substituteInPlace $out/share/paperless/paperless/settings.py \
--replace "localhost:8080" "http://localhost:8080"
'';
buildPhase = let
# Paperless has explicit runtime checks that expect these binaries to be in PATH
extraBin = lib.makeBinPath [ imagemagick ghostscript optipng tesseract unpaper ];
in ''
${python.interpreter} -m compileall $srcDir
makeWrapper $pythonEnv/bin/python $out/bin/paperless \
--set PATH ${extraBin} --add-flags $out/share/paperless/manage.py
# A shell snippet that can be sourced to setup a paperless env
cat > $out/share/paperless/setup-env.sh <<EOF
export PATH="$pythonEnv/bin:${extraBin}''${PATH:+:}$PATH"
export paperlessSrc=$out/share/paperless
EOF
'';
checkPhase = ''
source $out/share/paperless/setup-env.sh
tmpDir=$(realpath testsTmp)
mkdir $tmpDir
export HOME=$tmpDir
export PAPERLESS_MEDIADIR=$tmpDir
cd $paperlessSrc
# Prevent tests from writing to the derivation output
chmod -R -w $out
# Disable cache to silence a pytest warning ("could not create cache")
$pythonCheckEnv/bin/pytest -p no:cacheprovider
'';
passthru = {
withConfig = callPackage ./withConfig.nix {};
inherit python runtimePackages checkPackages tesseract;
};
meta = with lib; {
description = "Scan, index, and archive all of your paper documents";
homepage = "https://github.com/the-paperless-project/paperless";
license = licenses.gpl3;
maintainers = [ maintainers.earvstedt ];
};
};
python = python3.override {
packageOverrides = self: super: let
customPkgs = import ./python-modules super fetchFromGitHub; in
{
pyocr = pyocrWithUserTesseract super;
# Paperless is incompatible with factory_boy >= 3
factory_boy = customPkgs.factory_boy_2_12_0;
# These are pre-release versions, hence they are private to this pkg
django-filter = self.callPackage ./python-modules/django-filter.nix {};
django-crispy-forms = self.callPackage ./python-modules/django-crispy-forms.nix {};
};
};
runtimePackages = with python.pkgs; [
dateparser
python-dateutil
django
django-cors-headers
django-crispy-forms
django-filter
django_extensions
djangoql
djangorestframework
factory_boy
filemagic
fuzzywuzzy
langdetect
pdftotext
pillow
psycopg2
pyocr
python-dotenv
python-gnupg
pytz
termcolor
] ++ (lib.optional stdenv.isLinux inotify-simple);
checkPackages = with python.pkgs; [
pytest
pytest-django
pytest-env
pytest-xdist
];
pyocrWithUserTesseract = pyPkgs:
let
pyocr = pyPkgs.pyocr.override { inherit tesseract; };
in
if pyocr.outPath == pyPkgs.pyocr.outPath then
pyocr
else
# The user has provided a custom tesseract derivation that might be
# missing some languages that are required for PyOCR's tests. Disable them to
# avoid build errors.
pyocr.overridePythonAttrs (attrs: {
doCheck = false;
});
in
paperless
@@ -1,11 +0,0 @@
pyPkgs: fetchFromGitHub:
{
factory_boy_2_12_0 = pyPkgs.factory_boy.overridePythonAttrs (old: rec {
version = "2.12.0";
src = pyPkgs.fetchPypi {
inherit (old) pname;
inherit version;
sha256 = "0w53hjgag6ad5i2vmrys8ysk54agsqvgbjy9lg8g0d8pi9h8vx7s";
};
});
}
@@ -1,39 +0,0 @@
{ lib
, buildPythonPackage
, fetchFromGitHub
, pytestCheckHook
, pytest-django
, django
}:
buildPythonPackage rec {
pname = "django-crispy-forms";
version = "1.10.0";
src = fetchFromGitHub {
owner = "django-crispy-forms";
repo = "django-crispy-forms";
rev = version;
sha256 = "0y6kskfxgckb9npcgwx4zrs5n9px159zh9zhinhxi3i7wlriqpf5";
};
# For reasons unknown, the source dir must contain a dash
# for the tests to run successfully
postUnpack = ''
mv $sourceRoot source-
export sourceRoot=source-
'';
checkInputs = [ django pytest-django pytestCheckHook ];
preCheck = ''
export DJANGO_SETTINGS_MODULE=crispy_forms.tests.test_settings
'';
meta = with lib; {
description = "The best way to have DRY Django forms";
homepage = "https://github.com/maraujop/django-crispy-forms";
license = licenses.mit;
maintainers = with maintainers; [ earvstedt ];
};
}
@@ -1,26 +0,0 @@
{ lib, buildPythonPackage, python, pythonOlder, fetchFromGitHub
, django, django-crispy-forms, djangorestframework, mock, pytz }:
buildPythonPackage rec {
pname = "django-filter";
version = "2.1.0-pre";
disabled = pythonOlder "3.4";
src = fetchFromGitHub {
owner = "carltongibson";
repo = pname;
rev = "24adad8c48bc9e7c7539b6510ffde4ce4effdc29";
sha256 = "0hv4w95jnlzp9vdximl6bb27fyi75001jhvsbs0ikkd8amq8iaj7";
};
checkInputs = [ django django-crispy-forms djangorestframework mock pytz ];
checkPhase = "${python.interpreter} runtests.py";
meta = with lib; {
description = "A reusable Django application for allowing users to filter querysets dynamically.";
homepage = "https://github.com/carltongibson/django-filter";
license = licenses.bsd3;
maintainers = with maintainers; [ earvstedt ];
};
}
@@ -1,68 +0,0 @@
{ paperless, lib, writers }:
## Usage
#
# nix-build --out-link ./paperless -E '
# (import <nixpkgs> {}).paperless.withConfig {
# dataDir = /tmp/paperless-data;
# config = {
# PAPERLESS_DISABLE_LOGIN = "true";
# };
# }'
#
# Setup DB
# ./paperless migrate
#
# Consume documents in ${dataDir}/consume
# ./paperless document_consumer --oneshot
#
# Start web interface
# ./paperless runserver --noreload localhost:8000
{ config ? {}, dataDir ? null, ocrLanguages ? null
, paperlessPkg ? paperless, extraCmds ? "" }:
with lib;
let
paperless = if ocrLanguages == null then
paperlessPkg
else
(paperlessPkg.override {
tesseract = paperlessPkg.tesseract.override {
enableLanguages = ocrLanguages;
};
}).overrideDerivation (_: {
# `ocrLanguages` might be missing some languages required by the tests.
doCheck = false;
});
envVars = (optionalAttrs (dataDir != null) {
PAPERLESS_CONSUMPTION_DIR = "${dataDir}/consume";
PAPERLESS_MEDIADIR = "${dataDir}/media";
PAPERLESS_STATICDIR = "${dataDir}/static";
PAPERLESS_DBDIR = dataDir;
}) // config;
envVarDefs = mapAttrsToList (n: v: ''export ${n}="${toString v}"'') envVars;
setupEnvVars = builtins.concatStringsSep "\n" envVarDefs;
setupEnv = ''
source ${paperless}/share/paperless/setup-env.sh
${setupEnvVars}
${optionalString (dataDir != null) ''
mkdir -p "$PAPERLESS_CONSUMPTION_DIR" \
"$PAPERLESS_MEDIADIR" \
"$PAPERLESS_STATICDIR" \
"$PAPERLESS_DBDIR"
''}
'';
runPaperless = writers.writeBash "paperless" ''
set -e
${setupEnv}
${extraCmds}
exec python $paperlessSrc/manage.py "$@"
'';
in
runPaperless // {
inherit paperless setupEnv;
}
@@ -0,0 +1,28 @@
{ lib
, buildPythonPackage
, fetchPypi
, inotify-simple
}:
buildPythonPackage rec {
pname = "inotifyrecursive";
version = "0.3.5";
src = fetchPypi {
inherit pname version;
sha256 = "osRQsxdpPkU4QW+Q6x14WFBtr+a4uIUDe9LdmuLa+h4=";
};
propagatedBuildInputs = [ inotify-simple ];
# No tests included
doCheck = false;
pythonImportsCheck = [ pname ];
meta = with lib; {
description = "Simple recursive inotify watches for Python";
homepage = "https://github.com/letorbi/inotifyrecursive";
license = licenses.lgpl3Plus;
maintainers = with maintainers; [ Flakebi ];
};
}
@@ -0,0 +1,29 @@
{ lib
, buildPythonPackage
, fetchPypi
, pyyaml
, requests
}:
buildPythonPackage rec {
pname = "tika";
version = "1.24";
src = fetchPypi {
inherit pname version;
sha256 = "wsUPQFYi90UxhBEE+ehcF1Ea7eEd6OU4XqsaKaMfGRs=";
};
propagatedBuildInputs = [ pyyaml requests ];
# Requires network
doCheck = false;
pythonImportsCheck = [ pname ];
meta = with lib; {
description = "A Python binding to the Apache Tika REST services";
homepage = "https://github.com/chrismattmann/tika-python";
license = licenses.asl20;
maintainers = with maintainers; [ Flakebi ];
};
}
+1
View File
@@ -567,6 +567,7 @@ mapAliases ({
owncloudclient = owncloud-client; # added 2016-08
ocz-ssd-guru = throw "ocz-ssd-guru has been removed due to there being no source available"; # added 2021-07-12
p11_kit = p11-kit; # added 2018-02-25
paperless = paperless-ng; # added 2021-06-06
parity = openethereum; # added 2020-08-01
parquet-cpp = arrow-cpp; # added 2018-09-08
pass-otp = pass.withExtensions (ext: [ext.pass-otp]); # added 2018-05-04
+1 -1
View File
@@ -7893,7 +7893,7 @@ with pkgs;
pamtester = callPackage ../tools/security/pamtester { };
paperless = callPackage ../applications/office/paperless { };
paperless-ng = callPackage ../applications/office/paperless-ng { };
paperwork = callPackage ../applications/office/paperwork/paperwork-gtk.nix { };
+4
View File
@@ -3585,6 +3585,8 @@ in {
inotify-simple = callPackage ../development/python-modules/inotify-simple { };
inotifyrecursive = callPackage ../development/python-modules/inotifyrecursive { };
inquirer = callPackage ../development/python-modules/inquirer { };
intake = callPackage ../development/python-modules/intake { };
@@ -8746,6 +8748,8 @@ in {
tifffile = callPackage ../development/python-modules/tifffile { };
tika = callPackage ../development/python-modules/tika { };
tiledb = callPackage ../development/python-modules/tiledb {
inherit (pkgs) tiledb;
};