From a8c7c8e1be7a91bb67151f793a2d40a521711344 Mon Sep 17 00:00:00 2001 From: phanirithvij Date: Tue, 20 Jan 2026 15:15:01 +0530 Subject: [PATCH 1/3] pdfding: init at 1.4.1 Co-Authored-By: eljamm Signed-off-by: phanirithvij --- pkgs/by-name/pd/pdfding/frontend.nix | 87 +++++++++++ pkgs/by-name/pd/pdfding/package.nix | 214 +++++++++++++++++++++++++++ pkgs/by-name/pd/pdfding/update.sh | 53 +++++++ 3 files changed, 354 insertions(+) create mode 100644 pkgs/by-name/pd/pdfding/frontend.nix create mode 100644 pkgs/by-name/pd/pdfding/package.nix create mode 100755 pkgs/by-name/pd/pdfding/update.sh diff --git a/pkgs/by-name/pd/pdfding/frontend.nix b/pkgs/by-name/pd/pdfding/frontend.nix new file mode 100644 index 000000000000..753db30af5e6 --- /dev/null +++ b/pkgs/by-name/pd/pdfding/frontend.nix @@ -0,0 +1,87 @@ +{ + stdenv, + nodejs, + npmHooks, + fetchpatch2, + fetchNpmDeps, + fetchzip, + fetchFromGitHub, + tailwindcss_4, +}: +let + pdfjsVersion = "5.4.296"; # see update script + pdfjsHash = "sha256-b4W7wETq2CIZm2rJCmXEYvPhQtCbXS76L7GDvng6wn4="; + pdfjs = fetchzip { + url = "https://github.com/mozilla/pdf.js/releases/download/v${pdfjsVersion}/pdfjs-${pdfjsVersion}-dist.zip"; + hash = pdfjsHash; + stripRoot = false; + postFetch = '' + rm -rf $out/web/locale \ + $out/web/standard_fonts \ + $out/web/compressed.tracemonkey-pldi-09.pdf + + # remove source maps + find "$out" -name '*.map' -exec rm -f '{}' \; + ''; + }; +in +stdenv.mkDerivation (finalAttrs: { + pname = "pdfding-frontend"; + version = "1.4.1"; + src = fetchFromGitHub { + owner = "mrmn2"; + repo = "PdfDing"; + tag = "v${finalAttrs.version}"; + hash = "sha256-8e80gMdB6U3977dIU7bIAAEguYmi+AWQgUgYPDLCYLI="; + }; + + npmDeps = fetchNpmDeps { + inherit (finalAttrs) src; + name = "pdfding-frontend-${finalAttrs.version}-npm-deps"; + hash = "sha256-v1NFqDnFcRK8sd0bV3ck+LLMYQ90Dl1R1OnBTwWUVUg="; + }; + + patches = [ + # remove patch in 1.4.2 + # package.json has missing version and name + (fetchpatch2 { + url = "https://github.com/mrmn2/PdfDing/pull/203.patch?full_index=1"; + hash = "sha256-lKtpqKdyoGZdU4fTegto+YUIduIWbM82RQU9459NpC0="; + }) + ]; + + nativeBuildInputs = [ + nodejs + npmHooks.npmConfigHook + # it is in package.json and thus node_modules but no cli executable + tailwindcss_4 + ]; + + # keeping the file structure same as upstream to minimise confusion + buildPhase = '' + runHook preBuild + mkdir -p $out/pdfding + cp -r --no-preserve=mode pdfding/static $out/pdfding/static + cp -r --no-preserve=mode ${finalAttrs.passthru.pdfjs} $out/pdfding/static/pdfjs + + tailwindcss -i $out/pdfding/static/css/input.css -o $out/pdfding/static/css/tailwind.css --minify + rm $out/pdfding/static/css/input.css + + for i in build/pdf.mjs build/pdf.sandbox.mjs build/pdf.worker.mjs web/viewer.mjs; \ + do node_modules/terser/bin/terser $out/pdfding/static/pdfjs/$i --compress -o $out/pdfding/static/pdfjs/$i; done + + npm run build + + cp -r pdfding/static/js $out/pdfding/static + + runHook postBuild + ''; + + passthru = { + inherit pdfjs; + }; + + meta = { + description = "PdfDing frontend"; + }; +}) diff --git a/pkgs/by-name/pd/pdfding/package.nix b/pkgs/by-name/pd/pdfding/package.nix new file mode 100644 index 000000000000..7e7715851396 --- /dev/null +++ b/pkgs/by-name/pd/pdfding/package.nix @@ -0,0 +1,214 @@ +{ + lib, + python3, + callPackage, + fetchFromGitHub, + fetchpatch2, + makeWrapper, +}: +let + python = python3; +in +python.pkgs.buildPythonPackage (finalAttrs: { + pname = "pdfding"; + version = "1.4.1"; + src = fetchFromGitHub { + owner = "mrmn2"; + repo = "PdfDing"; + tag = "v${finalAttrs.version}"; + hash = "sha256-rrUaqxDO16NAOic74jeYgN+7Alvo+fIIacJdSOg0hFM="; + # remove in 1.5.0 + postFetch = "mv $out/{license.txt,LICENSE}"; + }; + pyproject = true; + + patches = [ + # remove all patches in 1.5.0 (next version after 1.4.1) + # patch to add data_dir + (fetchpatch2 { + url = "https://github.com/mrmn2/PdfDing/commit/f4945f2836ca8d972fcee2f00ef1d9cf217bada1.patch?full_index=1"; + hash = "sha256-VGjyIAVi+qd2WZ8FVKKC2ijLinoflO7RmPwIW1/oGcY="; + }) + # pyproject.toml still has 0.1.1 very old version + (fetchpatch2 { + url = "https://github.com/mrmn2/PdfDing/pull/203.patch?full_index=1"; + hash = "sha256-lKtpqKdyoGZdU4fTegto+YUIduIWbM82RQU9459NpC0="; + }) + # allows customising consume_schedule crontab + (fetchpatch2 { + url = "https://github.com/mrmn2/PdfDing/commit/96a13574718e0d27240eee8893fb799a02f24c05.patch?full_index=1"; + hash = "sha256-Stq392rIbsphvaE23GgFWb91KzpD6aOQu9MGDDoaO7s="; + }) + # allow specifying region for s3 backups + (fetchpatch2 { + url = "https://github.com/mrmn2/PdfDing/commit/3e412654f62d83b745111bd1d3587aca1a7739e1.patch?full_index=1"; + hash = "sha256-RQS3yJjrIaViFlm6it6zyRZOn+nTQE8qr8OpY+zYSCY="; + }) + ]; + + # remove supervisor from dependencies + postPatch = '' + sed -i 's/supervisor.*$//' pyproject.toml + + substituteInPlace pdfding/backup/tests/test_management.py pdfding/backup/tests/test_tasks.py \ + --replace-fail "Path(__file__).parents[2]" "Path('$PDFDING_OUT_DIR')" + ''; + + dependencies = + with python.pkgs; + [ + django + django-allauth + django-cleanup + django-htmx + gunicorn + markdown + minio + nh3 + psycopg2-binary + pypdf + pypdfium2 + python-magic + qrcode + rapidfuzz + ruamel-yaml + whitenoise + huey + pillow + oauthlib + + # dependecies required for django collectstatic + requests + pyjwt + cryptography + ] + ++ qrcode.optional-dependencies.pil + ++ django-allauth.optional-dependencies.socialaccount; + + build-system = with python.pkgs; [ poetry-core ]; + + nativeBuildInputs = [ + makeWrapper + ]; + + optional-dependencies = { + e2e = with python.pkgs; [ + pytest + pytest-django + pytest-playwright + pytest-rerunfailures # required to retry some flaky e2e tests + ]; + }; + + preBuild = '' + # remove originals, copy from frontend + rm -rf pdfding/static + ln -s ${finalAttrs.passthru.frontend}/pdfding/static pdfding/static + + # staticfiles step requires prod configuration, remove dev.py + mv pdfding/core/settings/dev.py dev.py.bak + + ${python.pythonOnBuildForHost.interpreter} pdfding/manage.py collectstatic + + # not needed, now we have staticfiles directory + rm -rf pdfding/static + + # the following is from upstream's Dockerfile + + # remove django md5 hash from filenames of pdfjs as it will mess up the relative imports because of the whitenoise setup + export PDFJS_PATH="pdfding/staticfiles/pdfjs" + for file_name in $(find $PDFJS_PATH -type f -not -path "$PDFJS_PATH/web/images/*") + do + if [[ $file_name =~ "LICENSE" ]]; then + new=$(echo "$file_name" | sed -E "s/LICENSE\.[a-zA-Z0-9]{12}/LICENSE/"); + else + new=$(echo "$file_name" | sed -E "s/\.[a-zA-Z0-9]{12}\./\./"); + fi; + mv -- "$file_name" "$new"; + done \ + && echo 'Successfully removed hash from pdfjs files' + + echo "VERSION = '${finalAttrs.version}'" > pdfding/core/settings/version.py; + ''; + + env.PDFDING_OUT_DIR = "${placeholder "out"}/${python.sitePackages}/pdfding"; + + makeWrapperArgs = [ + "--set-default DATA_DIR /var/lib/pdfding" + # allow for gunicorn processes to have access to Python packages + "--prefix PYTHONPATH : " + "${python.pkgs.makePythonPath finalAttrs.passthru.dependencies}:${finalAttrs.env.PDFDING_OUT_DIR}" + ]; + + postInstall = '' + mkdir -p $out/bin + + makeWrapper "$PDFDING_OUT_DIR/manage.py" $out/bin/pdfding-manage \ + $makeWrapperArgs + + makeWrapper ${lib.getExe python.pkgs.gunicorn} $out/bin/pdfding-start \ + --add-flags '--bind ''${HOST_IP:-127.0.0.1}:''${HOST_PORT:-8080} core.wsgi:application' \ + $makeWrapperArgs + ''; + + pythonRelaxDeps = [ + "django" + "django-allauth" + "huey" + "minio" + "pypdf" + "pypdfium2" + ]; + + nativeCheckInputs = with python.pkgs; [ + pytest-django + pytestCheckHook + ]; + + # from .github/workflows/tests.yaml + pytestFlags = [ + "--ignore=e2e" + ]; + + /* + fix two breaking tests by providing full out path + AssertionError: Calls not found + AssertionError: 'add_file_to_minio' does not contain all of ... + */ + preCheck = '' + # dev.py is required for tests, restore it + mv dev.py.bak $PDFDING_OUT_DIR/core/settings/dev.py + + # tests should run in pdfding directory + pushd pdfding + ''; + + postCheck = '' + # come out of the pdfding directory + popd + + # remove dev.py + rm $PDFDING_OUT_DIR/core/settings/dev.py + ''; + + pythonImportsCheck = [ + "pdfding" + ]; + + passthru = { + inherit python; + frontend = callPackage ./frontend.nix { }; + updateScript = ./update.sh; + }; + + meta = { + changelog = "https://github.com/mrmn2/PdfDing/blob/${finalAttrs.src.rev}/CHANGELOG.md"; + description = "Selfhosted PDF manager, viewer and editor offering a seamless user experience on multiple devices"; + downloadPage = "https://github.com/mrmn2/PdfDing"; + homepage = "https://pdfding.com"; + license = lib.licenses.agpl3Only; + mainProgram = "pdfding-manage"; + maintainers = with lib.maintainers; [ phanirithvij ]; + teams = with lib.teams; [ ngi ]; + }; +}) diff --git a/pkgs/by-name/pd/pdfding/update.sh b/pkgs/by-name/pd/pdfding/update.sh new file mode 100755 index 000000000000..debf815b4e70 --- /dev/null +++ b/pkgs/by-name/pd/pdfding/update.sh @@ -0,0 +1,53 @@ +#!/usr/bin/env nix-shell +#!nix-shell -i bash -p curl jq nix-update gitMinimal prefetch-npm-deps coreutils + +set -x +set -eou pipefail + +version=$(curl ${GITHUB_TOKEN:+ -H "Authorization: Bearer $GITHUB_TOKEN"} -sL https://api.github.com/repos/mrmn2/PdfDing/releases/latest | jq -r '.tag_name') + +if [[ "v${UPDATE_NIX_OLD_VERSION:-}" == "$version" ]]; then + echo "Already up-to-date, version: $version" + exit 0 +fi + +# source hashes +nix-update --version="$version" pdfding +nix-update --version="$version" pdfding.frontend + +NIXPKGS_PATH="$(git rev-parse --show-toplevel)" +PACKAGE_DIR="$( cd -- "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )" + +TMPDIR="$(mktemp -d)" +trap 'rm -rf "$TMPDIR"' EXIT +cd "$TMPDIR" + +src="$(nix-build --no-link "$NIXPKGS_PATH" -A pdfding.src)" +cp "$src"/{package.json,package-lock.json} . + +# npmDeps hash +prev_npm_hash="$( + nix-instantiate "$NIXPKGS_PATH" \ + --eval --json \ + -A pdfding.frontend.npmDeps.hash | + jq -r . +)" +new_npm_hash="$(prefetch-npm-deps ./package-lock.json)" + +sed -i "s|$prev_npm_hash|$new_npm_hash|g" "$PACKAGE_DIR/frontend.nix" + +# pdfjs version +pdfjs_version="$(grep 'PDFJS_VERSION=' "$src/Dockerfile" | cut -d'=' -f2)" + +sed -i "s|pdfjsVersion = .*;|pdfjsVersion = \"$pdfjs_version\";|" "$PACKAGE_DIR/frontend.nix" + +# pdfjs hash +sed -i "s|pdfjsHash = .*;|pdfjsHash = \"sha256-AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=\";|" "$PACKAGE_DIR/frontend.nix" + +set +e +new_pdfjs_hash="$( + nix-build --no-out-link -A pdfding.frontend.pdfjs "$NIXPKGS_PATH" 2>&1 >/dev/null | grep "got:" | cut -d':' -f2 | sed 's| ||g' +)" +set -e + +sed -i "s|\"sha256-AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=\"|\"$new_pdfjs_hash\"|g" "$PACKAGE_DIR/frontend.nix" From 364039cf28043308b5c6eae3052747542dc75669 Mon Sep 17 00:00:00 2001 From: phanirithvij Date: Wed, 21 Jan 2026 20:19:42 +0530 Subject: [PATCH 2/3] nixos/pdfding: init Co-authored-by: Sandro Co-Authored-By: eljamm Signed-off-by: phanirithvij --- .../manual/release-notes/rl-2605.section.md | 2 + nixos/modules/module-list.nix | 1 + nixos/modules/services/web-apps/pdfding.nix | 451 ++++++++++++++++++ nixos/tests/all-tests.nix | 4 + nixos/tests/web-apps/pdfding/basic.nix | 165 +++++++ nixos/tests/web-apps/pdfding/default.nix | 7 + nixos/tests/web-apps/pdfding/e2e.nix | 81 ++++ nixos/tests/web-apps/pdfding/postgres.nix | 97 ++++ nixos/tests/web-apps/pdfding/s3-backups.nix | 201 ++++++++ pkgs/by-name/pd/pdfding/package.nix | 2 + 10 files changed, 1011 insertions(+) create mode 100644 nixos/modules/services/web-apps/pdfding.nix create mode 100644 nixos/tests/web-apps/pdfding/basic.nix create mode 100644 nixos/tests/web-apps/pdfding/default.nix create mode 100644 nixos/tests/web-apps/pdfding/e2e.nix create mode 100644 nixos/tests/web-apps/pdfding/postgres.nix create mode 100644 nixos/tests/web-apps/pdfding/s3-backups.nix diff --git a/nixos/doc/manual/release-notes/rl-2605.section.md b/nixos/doc/manual/release-notes/rl-2605.section.md index d1fe153f0df1..42def2d4b3be 100644 --- a/nixos/doc/manual/release-notes/rl-2605.section.md +++ b/nixos/doc/manual/release-notes/rl-2605.section.md @@ -18,6 +18,8 @@ - [ImmichFrame](https://immichframe.dev/), display your photos from Immich as a digital photo frame. Available as `services.immichframe`. +- [PdfDing](https://www.pdfding.com/), manage, view and edit your PDFs seamlessly on all your devices wherever you are. Available as [services.pdfding](#opt-services.pdfding.enable). + - [reaction](https://reaction.ppom.me/), a daemon that scans program outputs for repeated patterns, and takes action. A common usage is to scan ssh and webserver logs, and to ban hosts that cause multiple authentication errors. A modern alternative to fail2ban. Available as [services.reaction](#opt-services.reaction.enable). - [rqbit](https://github.com/ikatson/rqbit), a bittorrent client written in Rust. It has HTTP API and Web UI, and can be used as a library. Available as [services.rqbit](#opt-services.rqbit.enable). diff --git a/nixos/modules/module-list.nix b/nixos/modules/module-list.nix index cdd258e5b176..77fdbc9528b4 100644 --- a/nixos/modules/module-list.nix +++ b/nixos/modules/module-list.nix @@ -1711,6 +1711,7 @@ ./services/web-apps/outline.nix ./services/web-apps/pairdrop.nix ./services/web-apps/part-db.nix + ./services/web-apps/pdfding.nix ./services/web-apps/peering-manager.nix ./services/web-apps/peertube-runner.nix ./services/web-apps/peertube.nix diff --git a/nixos/modules/services/web-apps/pdfding.nix b/nixos/modules/services/web-apps/pdfding.nix new file mode 100644 index 000000000000..cb5c5e729139 --- /dev/null +++ b/nixos/modules/services/web-apps/pdfding.nix @@ -0,0 +1,451 @@ +{ + config, + lib, + pkgs, + ... +}: +let + inherit (lib) + concatStringsSep + mapAttrsToList + mkEnableOption + mkPackageOption + mkOption + optionalAttrs + optionalString + types + ; + + cfg = config.services.pdfding; + + stateDir = "/var/lib/pdfding"; + + usePostgres = cfg.database.type == "postgres"; + + envVars = { + # HOST_IP is used in the package derivation + HOST_IP = cfg.hostName; + HOST_NAME = concatStringsSep "," cfg.allowedHosts; + HOST_PORT = toString cfg.port; + DATABASE_TYPE = ""; + DATA_DIR = stateDir; + } + // optionalAttrs usePostgres { + DATABASE_TYPE = "POSTGRES"; + POSTGRES_PORT = toString cfg.database.port; + # Django Uses the unix domain socket + # if host is set to empty see https://docs.djangoproject.com/en/6.0/ref/settings/#host + POSTGRES_HOST = lib.optionalString (!cfg.database.createLocally) cfg.database.host; + POSTGRES_NAME = cfg.database.name; + POSTGRES_USER = cfg.database.user; + } + // optionalAttrs cfg.consume.enable { + CONSUME_ENABLE = "TRUE"; + CONSUME_SCHEDULE = cfg.consume.schedule; + } + // optionalAttrs cfg.backup.enable { + BACKUP_ENABLE = "TRUE"; + BACKUP_ENDPOINT = cfg.backup.endpoint; + BACKUP_SCHEDULE = cfg.backup.schedule; + } + // cfg.extraEnvironment; + + envFile = pkgs.writeText "pdfding.env" ( + lib.pipe envVars [ + (mapAttrsToList (name: value: "${name}=\"${toString value}\"")) + (concatStringsSep "\n") + ] + ); + + loadCreds = + optionalString (usePostgres && !cfg.database.createLocally) '' + export POSTGRES_PASSWORD="$(<${cfg.database.passwordFile})" + '' + + '' + export SECRET_KEY="$(<${cfg.secretKeyFile})" + ''; + + secretRecommendation = "Consider using a secret managing scheme such as `agenix` or `sops-nix` to generate this file."; +in +{ + options.services.pdfding = { + enable = mkEnableOption "PdfDing service" // { + description = '' + Whether to enable PdfDing service. + + To use the pdfding-manage CLI, add your user to the pdfding group: + users.users..extraGroups = [ "pdfding" ]; + ''; + }; + + package = mkPackageOption pkgs "pdfding" { }; + + user = mkOption { + type = types.str; + default = "pdfding"; + description = "User account under which PdfDing runs"; + }; + + group = mkOption { + type = types.str; + default = "pdfding"; + description = "Group under which PdfDing runs"; + }; + + hostName = mkOption { + type = types.str; + default = "0.0.0.0"; + example = "pdfding.example.com"; + description = "Listen address for PdfDing"; + }; + + port = mkOption { + type = types.port; + default = 8000; + description = "Port on which PdfDing listens"; + }; + + allowedHosts = mkOption { + type = types.listOf types.str; + default = [ + "127.0.0.1" + "localhost" + ]; + description = "Domains where PdfDing is allowed to run"; + }; + + gunicorn.extraArgs = mkOption { + type = types.listOf types.str; + description = "Command line arguments passed to Gunicorn server."; + default = [ ]; + }; + + extraEnvironment = mkOption { + type = types.attrsOf types.str; + default = { }; + description = "Additional environment variables"; + }; + + envFiles = mkOption { + type = types.listOf types.path; + description = "Environment variable files"; + default = [ ]; + }; + + secretKeyFile = mkOption { + type = types.path; + default = null; + description = "File containing the Django SECRET_KEY. ${secretRecommendation}"; + example = "/run/secrets/pdfding-secret-key"; + }; + + database = { + type = mkOption { + type = types.enum [ + "sqlite" + "postgres" + ]; + default = "sqlite"; + description = "Database type to use"; + }; + + host = mkOption { + type = types.str; + default = ""; + description = "PostgreSQL host"; + }; + + port = mkOption { + type = types.port; + default = 5432; + description = "PostgreSQL port"; + }; + + name = mkOption { + type = types.str; + default = "pdfding"; + description = "PostgreSQL database name"; + }; + + user = mkOption { + type = types.str; + default = "pdfding"; + description = "PostgreSQL user"; + }; + + passwordFile = mkOption { + type = types.nullOr types.path; + default = null; + description = "File containing POSTGRES_PASSWORD. ${secretRecommendation}"; + example = "/run/secrets/pdfding-db-password"; + }; + + createLocally = mkOption { + type = types.bool; + default = false; + description = "Whether to create a local PostgreSQL database automatically"; + }; + }; + + consume = { + enable = mkEnableOption "Consume functionality" // { + description = '' + Bulk PDF import from consume directory. + + When enabled, administrators can create per-user directories like /var/lib/pdfding/consume/ + with permissions allowing the pdfding user to read and write. + PDFs placed in these directories are automatically imported into user accounts. + + PDFs are imported periodically via cronjob and successfully imported files + are automatically deleted from the consume directory. + ''; + }; + schedule = mkOption { + type = types.str; + default = "*/5 * * * *"; + description = '' + The cron schedule for the consume task to trigger. + The format is "minute hour day month day_of_week" + Read + - https://github.com/mrmn2/PdfDing/blob/d0f21ec2f9fbee4b1a2f6b7e0e6c7ea7784ab1bc/pdfding/base/task_helpers.py#L5 + - https://huey.readthedocs.io/en/latest/api.html#crontab + ''; + }; + }; + + backup = { + enable = mkEnableOption "Backup functionality" // { + description = '' + Automatic backup of important data to a AWS S3 (or compatible) instance. + + When enabled and properly configured via environment variables, + important data is periodically uploaded to the specified s3 + instance via cronjob. + ''; + }; + schedule = mkOption { + type = types.str; + default = "0 2 * * *"; + description = '' + The cron schedule for the consume task to trigger. + The format is "minute hour day month day_of_week" + Read + - https://github.com/mrmn2/PdfDing/blob/d0f21ec2f9fbee4b1a2f6b7e0e6c7ea7784ab1bc/pdfding/base/task_helpers.py#L5 + - https://huey.readthedocs.io/en/latest/api.html#crontab + ''; + }; + endpoint = mkOption { + type = types.nullOr types.str; + default = null; + description = "The s3 endpoint for backups"; + example = "127.0.0.1:9000"; + }; + }; + + openFirewall = lib.mkOption { + type = types.bool; + default = false; + description = "Open ports in the firewall for the PdfDing web interface."; + }; + + installTestHelpers = mkOption { + type = types.bool; + default = false; + internal = true; + description = "Adds a few helper commands to systemPackages for nixos tests"; + }; + }; + + config = lib.mkIf cfg.enable { + assertions = [ + { + assertion = cfg.secretKeyFile != null; + message = "services.pdfding.secretKeyFile must be set when using PdfDing"; + } + { + assertion = cfg.backup.enable -> envVars.BACKUP_ENDPOINT != null; + message = "services.pdfding.extraEnvironment.BACKUP_ENDPOINT must be set when backup is enabled"; + } + { + assertion = cfg.database.createLocally -> usePostgres; + message = "services.pdfding.database.createLocally is enabled but not database.type is not postgres"; + } + { + assertion = cfg.database.createLocally -> cfg.database.host == ""; + message = "services.pdfding.database.host must be empty when services.pdfding.database.createLocally is enabled"; + } + { + assertion = cfg.database.createLocally -> cfg.database.passwordFile == null; + message = "specifying services.pdfding.database.passwordFile is not supported when used along with a local db setup"; + } + { + assertion = + cfg.database.createLocally + -> cfg.database.user == cfg.user && cfg.database.user == cfg.database.name; + message = "services.pdfding.database.user should be the same as services.pdfding.user as well as services.pdfding.database.name when running a local db setup"; + } + ]; + + networking.firewall = lib.mkIf cfg.openFirewall { + allowedTCPPorts = [ cfg.port ]; + }; + + environment.systemPackages = + let + genWrapper = + name: cmd: + pkgs.writeShellScriptBin name '' + set -eou pipefail + set -a + ${lib.toShellVars cfg.extraEnvironment} + ${lib.concatMapStringsSep "\n" (f: "source ${f}") cfg.envFiles} + set +a + ${loadCreds} + sudo=exec + if [[ "$USER" != ${cfg.user} ]]; then + sudo='${config.security.wrapperDir}/sudo -E -u ${cfg.user}' + fi + ${cmd} + ''; + commands.pdfding-manage = '' + $sudo ${lib.getExe cfg.package} "$@" + ''; + commands.consume-immediate = '' + echo "from pdf.tasks import consume_function; consume_function(True)" | \ + $sudo ${lib.getExe cfg.package} shell + ''; + commands.backup-immediate = '' + echo "from backup.tasks import backup_function; backup_function()" | \ + $sudo ${lib.getExe cfg.package} shell + ''; + packages = lib.genAttrs (lib.attrNames commands) (name: genWrapper name commands.${name}); + in + lib.mkMerge [ + [ + packages.pdfding-manage + ] + (lib.mkIf cfg.installTestHelpers [ + packages.consume-immediate + packages.backup-immediate + ]) + ]; + + users.users.${cfg.user} = { + isSystemUser = true; + group = cfg.group; + }; + + users.groups.${cfg.group} = { }; + + services.pdfding.envFiles = [ envFile ]; + + services.pdfding.extraEnvironment = { + DEFAULT_THEME = "dark"; + DEFAULT_THEME_COLOR = "green"; + }; + + services.pdfding.gunicorn.extraArgs = [ + "--workers=4" + "--max-requests=1200" + "--max-requests-jitter=50" + "--log-level=error" + ]; + + systemd.services.pdfding = { + description = "PdfDing Web Service"; + after = [ + "network.target" + ] + ++ lib.optionals (usePostgres && cfg.database.createLocally) [ "postgresql.target" ]; + wantedBy = [ "multi-user.target" ]; + + preStart = '' + ${loadCreds} + ${optionalString (usePostgres && cfg.database.createLocally) + # bash + '' + count=0 + timeout=30 + until ${pkgs.postgresql}/bin/pg_isready -p ${toString cfg.database.port}; do + if [ $count -ge $timeout ]; then + echo "Timed out waiting for PostgreSQL after $timeout seconds." + exit 1 + fi + echo "Waiting for PostgreSQL... ($count/$timeout)" + sleep 1 + count=$((count+1)) + done + '' + } + + ${cfg.package}/bin/pdfding-manage migrate + ${cfg.package}/bin/pdfding-manage clean_up + ''; + + script = '' + ${loadCreds} + exec ${cfg.package}/bin/pdfding-start ${toString cfg.gunicorn.extraArgs} + ''; + + serviceConfig = { + Type = "exec"; + User = cfg.user; + Group = cfg.group; + EnvironmentFile = cfg.envFiles; + StateDirectory = [ + "pdfding" + "pdfding/db" + "pdfding/media" + ] + ++ lib.optional cfg.consume.enable "pdfding/consume"; + NoNewPrivileges = true; + PrivateTmp = true; + PrivateDevices = true; + ProtectSystem = "strict"; + ProtectHome = true; + Restart = "on-failure"; + RestartSec = "5s"; + }; + }; + + systemd.services.pdfding-background = lib.mkIf (cfg.consume.enable || cfg.backup.enable) { + description = "PdfDing Background Tasks (Huey)"; + after = [ "pdfding.service" ]; + wantedBy = [ "multi-user.target" ]; + script = '' + ${loadCreds} + exec ${cfg.package}/bin/pdfding-manage run_huey + ''; + serviceConfig = { + Type = "exec"; + User = cfg.user; + Group = cfg.group; + WorkingDirectory = stateDir; + EnvironmentFile = cfg.envFiles; + + NoNewPrivileges = true; + PrivateTmp = true; + PrivateDevices = true; + ProtectSystem = "strict"; + ProtectHome = true; + ReadWritePaths = [ stateDir ]; + Restart = "on-failure"; + RestartSec = "5s"; + TimeoutStopSec = 30; + }; + }; + + services.postgresql = lib.mkIf cfg.database.createLocally { + enable = true; + ensureDatabases = [ cfg.database.name ]; + ensureUsers = [ + { + name = cfg.database.user; + ensureDBOwnership = true; + } + ]; + }; + }; + + meta.maintainers = lib.teams.ngi.members; +} diff --git a/nixos/tests/all-tests.nix b/nixos/tests/all-tests.nix index 20b4dc35995a..fce9eb2830fc 100644 --- a/nixos/tests/all-tests.nix +++ b/nixos/tests/all-tests.nix @@ -1227,6 +1227,10 @@ in pass-secret-service = runTest ./pass-secret-service.nix; password-option-override-ordering = runTest ./password-option-override-ordering.nix; patroni = handleTestOn [ "x86_64-linux" ] ./patroni.nix { }; + pdfding = import ./web-apps/pdfding { + inherit (pkgs) lib; + inherit runTest; + }; pdns-recursor = runTest ./pdns-recursor.nix; peerflix = runTest ./peerflix.nix; peering-manager = runTest ./web-apps/peering-manager.nix; diff --git a/nixos/tests/web-apps/pdfding/basic.nix b/nixos/tests/web-apps/pdfding/basic.nix new file mode 100644 index 000000000000..9b1db591dfb5 --- /dev/null +++ b/nixos/tests/web-apps/pdfding/basic.nix @@ -0,0 +1,165 @@ +{ + lib, + pkgs, + ... +}: +{ + name = "PdfDing sqlite"; + + nodes = { + machine = + { ... }: + { + # WARNING: Do not add secrets to the world-readable /nix/store in a production deployment + # Use a secret management scheme instead https://wiki.nixos.org/wiki/Comparison_of_secret_managing_schemes + services.pdfding = { + enable = true; + secretKeyFile = pkgs.writeText "secretKeyFile" "test123"; + }; + + environment.systemPackages = with pkgs; [ + sqlite + ]; + + # test email validation works + services.pdfding.extraEnvironment = { + EMAIL_BACKEND = "SMTP"; + SMTP_HOST = "localhost"; + SMTP_PORT = "1025"; + SMTP_USER = ""; # mailpit doesn't need auth + SMTP_PASSWORD = ""; + SMTP_USE_TLS = "FALSE"; + SMTP_USE_SSL = "FALSE"; + }; + + # enable mailpit + services.mailpit.instances.default = { }; + }; + }; + + # Test the most basic user functionality expected from pdfding. + # Heavy e2e test suite is implemented in e2e.nix + testScript = + { nodes, ... }: + let + inherit (nodes.machine.services.pdfding) port; + mailpitApiEndpoint = "http://${nodes.machine.services.mailpit.instances.default.listen}/api/v1"; + stateDir = "/var/lib/pdfding"; + in + # py + '' + import json + from pprint import pprint + + # start vms + start_all() + + # create admin + machine.wait_for_unit("multi-user.target") + machine.succeed("DJANGO_SUPERUSER_PASSWORD=admin pdfding-manage createsuperuser --no-input --username admin --email admin@localhost") + + cookie_jar = "/tmp/cookies.txt" + endpoint = "http://localhost:${toString port}" + + with subtest("login and basic usage"): + # login + machine.succeed(f""" + curl -f \ + -X POST -c {cookie_jar} -b {cookie_jar} \ + -d "csrfmiddlewaretoken=$(curl -f -c {cookie_jar} -s '{endpoint}/accountlogin/' | grep -oP 'name="csrfmiddlewaretoken" value="\\K[^"]+')" \ + -d "login=admin@localhost" \ + -d "password=admin" \ + {endpoint}/accountlogin/ + """) + + test_pdf = "${pkgs.pdfding.src}/pdfding/pdf/tests/data/dummy.pdf" + + # verify no pdfs exist in db + machine.succeed("sqlite3 ${stateDir}/db/db.sqlite3 'SELECT COUNT(*) FROM pdf_pdf' | grep -q '^0$'") + + # upload + machine.succeed(f""" + csrf_token=$(curl -f -b {cookie_jar} -c {cookie_jar} -s "{endpoint}/pdf/add" | grep -oP 'name="csrfmiddlewaretoken" value="\\K[^"]+') + curl -f \ + -c {cookie_jar} -b {cookie_jar} \ + -F "notes=" \ + -F "tag_string=" \ + -F "description=" \ + -F "use_file_name=on" \ + -F "name=test-upload" \ + -F "file=@{test_pdf};type=application/pdf" \ + -F "csrfmiddlewaretoken=$csrf_token" \ + -H "Referer: {endpoint}/pdf/add" \ + {endpoint}/pdf/add + """) + + # download + machine.succeed(f""" + pdf_id=$(curl -f -b {cookie_jar} -s "{endpoint}/pdf/" | grep -oP 'href="/pdf/view/\\K[^"]+' | head -1) + curl -f -b {cookie_jar} -o /tmp/downloaded.pdf "{endpoint}/pdf/download/$pdf_id" + """) + + # verify pdf in user's dir + machine.succeed("test -f ${stateDir}/media/1/pdf/*.pdf") + + # verify one entry exists in sqlite db + machine.succeed("sqlite3 ${stateDir}/db/db.sqlite3 'SELECT COUNT(*) FROM pdf_pdf' | grep -q '^1$'") + + with subtest("email validation"): + # check we can reach mailpit + machine.succeed("curl -f ${mailpitApiEndpoint}/info") + + # check that no emails exist + result = json.loads(machine.succeed("curl -sf ${mailpitApiEndpoint}/messages")) + pprint(result) + assert result["total"] == 0 + + # signup + machine.succeed(f""" + curl -f \ + -X POST -c {cookie_jar} -b {cookie_jar} \ + -d "csrfmiddlewaretoken=$(curl -f -c {cookie_jar} -s '{endpoint}/accountsignup/' | grep -oP 'name="csrfmiddlewaretoken" value="\\K[^"]+')" \ + -d "email=pdfding_new_user@example.com" \ + -d "password1=foobarbaz" \ + -d "password2=foobarbaz" \ + {endpoint}/accountsignup/ + """) + + # wait a bit for email to be processed + machine.sleep(3) + + # verify the email was received by mailpit + result = json.loads(machine.succeed("curl -s ${mailpitApiEndpoint}/messages")) + pprint(result) + assert result["total"] == 1 + assert result["messages"][0]["To"][0]["Address"] == "pdfding_new_user@example.com" + ''; + + # Debug interactively with: + # - nix run .#nixosTests.pdfding.basic.driverInteractive -L + # - start_all() / run_tests() + interactive.sshBackdoor.enable = true; # ssh -o User=root vsock%3 + interactive.nodes.machine = + { config, ... }: + let + port = config.services.pdfding.port; + in + { + # not needed, only for manual interactive debugging + virtualisation.memorySize = 4096; + environment.systemPackages = with pkgs; [ + htop + ]; + + virtualisation.forwardPorts = map (port: { + from = "host"; + host.port = port; + guest.port = port; + }) [ port ]; + + # forwarded ports need to be accessible + networking.firewall.allowedTCPPorts = [ port ]; + }; + + meta.maintainers = lib.teams.ngi.members; +} diff --git a/nixos/tests/web-apps/pdfding/default.nix b/nixos/tests/web-apps/pdfding/default.nix new file mode 100644 index 000000000000..c819ca875db0 --- /dev/null +++ b/nixos/tests/web-apps/pdfding/default.nix @@ -0,0 +1,7 @@ +{ lib, runTest }: +lib.recurseIntoAttrs { + basic = runTest ./basic.nix; + e2e = runTest ./e2e.nix; + postgres = runTest ./postgres.nix; + s3-backups = runTest ./s3-backups.nix; +} diff --git a/nixos/tests/web-apps/pdfding/e2e.nix b/nixos/tests/web-apps/pdfding/e2e.nix new file mode 100644 index 000000000000..23bfd064c384 --- /dev/null +++ b/nixos/tests/web-apps/pdfding/e2e.nix @@ -0,0 +1,81 @@ +{ + lib, + pkgs, + ... +}: +{ + name = "PdfDing e2e tests"; + + nodes = { + machine = + { ... }: + { + environment.systemPackages = [ + (pkgs.writeShellScriptBin "tests_e2e" ( + let + pythonPath = + with pkgs.python3Packages; + [ + playwright + pkgs.pdfding + ] + ++ pkgs.pdfding.dependencies + ++ pkgs.pdfding.optional-dependencies.e2e; + in + # bash + '' + export PLAYWRIGHT_BROWSERS_PATH=${pkgs.playwright-driver.browsers} + export PYTHONPATH=${pkgs.python3Packages.makePythonPath pythonPath} + export PATH=${pkgs.pdfding.python}/bin:$PATH + export E2E_TESTS=1 + cd $(mktemp -d) + cp -r --no-preserve=all ${pkgs.pdfding.src} source + cd source + cp -ru --no-preserve=all ${pkgs.pdfding.frontend}/pdfding/static pdfding + # some tests are flaky due to timeouts, re-run them + python -m pytest pdfding/e2e \ + -x -r aR \ + --reruns 10 --only-rerun TimeoutError + '' + # see https://github.com/MrBin99/django-vite/issues/95 + # tdlr; collectstatic is not important for e2e tests which uses StaticLiveServerTestCase + # it only cares about files in static/ + )) + ]; + }; + }; + + testScript = + { nodes, ... }: + # py + '' + # start + start_all() + machine.wait_for_unit("multi-user.target") + + machine.succeed("tests_e2e | systemd-cat") + ''; + + # Debug interactively with: + # - nix run .#nixosTests.pdfding.e2e.driverInteractive -L + # - start_all() / run_tests() + interactive.sshBackdoor.enable = true; # ssh -o User=root vsock%3 + interactive.nodes.machine = + { config, ... }: + { + imports = [ + # enable graphical session + users (alice, bob) + ./common/x11.nix + ./common/user-account.nix + ]; + services.xserver.enable = true; + test-support.displayManager.auto.user = "alice"; + virtualisation.memorySize = lib.mkForce 8192; + environment.systemPackages = with pkgs; [ + htop + ]; + # env DISPLAY=:0 sudo -u alice tests_e2e | systemd-cat + }; + + meta.maintainers = lib.teams.ngi.members; +} diff --git a/nixos/tests/web-apps/pdfding/postgres.nix b/nixos/tests/web-apps/pdfding/postgres.nix new file mode 100644 index 000000000000..12a079402cb7 --- /dev/null +++ b/nixos/tests/web-apps/pdfding/postgres.nix @@ -0,0 +1,97 @@ +{ + lib, + pkgs, + ... +}: +{ + name = "PdfDing postgres"; + + nodes = { + machine = + { config, ... }: + { + # WARNING: Do not add secrets to the world-readable /nix/store in a production deployment + # Use a secret management scheme instead https://wiki.nixos.org/wiki/Comparison_of_secret_managing_schemes + services.pdfding = { + enable = true; + secretKeyFile = pkgs.writeText "secretKeyFile" "test123"; + database.createLocally = true; + database.type = "postgres"; + consume.enable = true; # allows bulk importing pdf files from the backend + consume.schedule = "*/1 * * * *"; # once every minute + installTestHelpers = true; + }; + + environment.systemPackages = [ + config.services.postgresql.finalPackage + ]; + }; + }; + + # Tests the most basic user functionality expected from pdfding with postgres and consume feature + testScript = + { nodes, ... }: + let + stateDir = "/var/lib/pdfding"; + in + # py + '' + # start + start_all() + + # create admin + machine.wait_for_unit("multi-user.target") + + machine.succeed("DJANGO_SUPERUSER_PASSWORD=admin pdfding-manage createsuperuser --no-input --username admin --email admin@localhost") + + test_pdf = "${pkgs.pdfding.src}/pdfding/pdf/tests/data/dummy.pdf" + + # copy to consume dir + machine.succeed(f"sudo -u pdfding bash -c 'mkdir -p ${stateDir}/consume/1 && cp {test_pdf} ${stateDir}/consume/1/'") + + # check the file was copied + output = machine.succeed("sudo -u pdfding bash -c 'ls -l ${stateDir}/consume/1/'") + assert "dummy.pdf" in output, "dummy pdf file not found" + + # check there are no pdfs + machine.succeed("sudo -u pdfding psql -tAc 'SELECT COUNT(*) FROM pdf_pdf' | grep -q '^0$'") + + print(machine.succeed("realpath /run/current-system/sw/bin/consume-immediate")) + print(machine.succeed("consume-immediate")) + + # verify pdf is in user's dir, and removed from consume dir + machine.wait_for_file("${stateDir}/media/1/pdf/dummy.pdf") + machine.fail("test -f ${stateDir}/consume/1/pdf/dummy.pdf") + + # verify pdf is also in postgres db + machine.succeed("sudo -u pdfding psql -tAc 'SELECT COUNT(*) FROM pdf_pdf' | grep -q '^1$'") + ''; + + # Debug interactively with: + # - nix run .#nixosTests.pdfding.postgres.driverInteractive -L + # - start_all() / run_tests() + interactive.sshBackdoor.enable = true; # ssh -o User=root vsock%3 + interactive.nodes.machine = + { config, ... }: + let + port = config.services.pdfding.port; + in + { + # not needed, only for manual interactive debugging + virtualisation.memorySize = 4096; + environment.systemPackages = with pkgs; [ + htop + ]; + + virtualisation.forwardPorts = map (port: { + from = "host"; + host.port = port; + guest.port = port; + }) [ port ]; + + # forwarded ports need to be accessible + networking.firewall.allowedTCPPorts = [ port ]; + }; + + meta.maintainers = lib.teams.ngi.members; +} diff --git a/nixos/tests/web-apps/pdfding/s3-backups.nix b/nixos/tests/web-apps/pdfding/s3-backups.nix new file mode 100644 index 000000000000..b8214eb66b9a --- /dev/null +++ b/nixos/tests/web-apps/pdfding/s3-backups.nix @@ -0,0 +1,201 @@ +{ + lib, + pkgs, + ... +}: +let + # WARNING: Do not add secrets to the world-readable /nix/store in a production deployment + # Use a secret management scheme instead https://wiki.nixos.org/wiki/Comparison_of_secret_managing_schemes + + # note: In a production deployment use `garage key generate`, along with the steps specified in the getting started guide of garage. + # (sops-nix or agenix for configuring secrets for garage is out of scope here) + garageAccessKey = "GK0a0a0a0b0b0b0c0c0c0d0d0d"; + garageSecretKey = "0a0a0a0a0b0b0b0b0c0c0c0c0d0d0d0d1a1a1a1a1b1b1b1b1c1c1c1c1d1d1d1d"; + pdfding-s3-keys = pkgs.writeText "pdfding-s3-keys" '' + BACKUP_ACCESS_KEY=${garageAccessKey} + BACKUP_SECRET_KEY=${garageSecretKey} + ''; +in +{ + name = "PdfDing s3 backups"; + + nodes = { + machine = + { config, ... }: + { + services.pdfding = { + enable = true; + secretKeyFile = pkgs.writeText "secretKeyFile" "test123"; + + backup.enable = true; + backup.schedule = "*/1 * * * *"; + backup.endpoint = "[::]:3900"; + extraEnvironment.BACKUP_BUCKET_NAME = "pdfding-bucket"; + extraEnvironment.BACKUP_REGION = "garage"; + + envFiles = [ pdfding-s3-keys ]; + installTestHelpers = true; + }; + + # Setup a local garage service for the backup feature + # taken from garage nixosTest + services.garage = { + enable = true; + package = pkgs.garage_2; + settings = { + rpc_bind_addr = "[::]:3901"; + rpc_public_addr = "[::1]:3901"; + rpc_secret = "5c1915fa04d0b6739675c61bf5907eb0fe3d9c69850c83820f51b4d25d13868c"; + + s3_api = { + s3_region = "garage"; + api_bind_addr = "[::]:3900"; + root_domain = ".s3.garage"; + }; + + s3_web = { + bind_addr = "[::]:3902"; + root_domain = ".web.garage"; + index = "index.html"; + }; + + replication_factor = 1; + }; + }; + + # setup garage bucket and credentials + # note: The nixos module has no option to specify secrets declaratively + systemd.services.garage.postStart = '' + export PATH="$PATH:${config.services.garage.package}/bin" + + # wait for garage to be up + while ! garage status >/dev/null 2>&1; do sleep 2; done + + if ! garage bucket list | grep -q pdfding-bucket; then + garage layout assign -z dc1 -c 1G $(garage status | cut -d' ' -f1 | tail -1) + garage layout apply --version 1 + + # note: the key id should be valid (starts with `GK`, followed by 12 hex-encoded bytes) + # the secret key should be valid (composed of 32 hex-encoded bytes) + garage key import ${garageAccessKey} ${garageSecretKey} -n pdfding-key --yes + + garage bucket create pdfding-bucket + + garage bucket allow --read --write --owner pdfding-bucket --key pdfding-key + fi + ''; + + # Garage requires at least 1GiB of free disk space to run. + virtualisation.diskSize = 2 * 1024; + + environment.systemPackages = with pkgs; [ + minio-client + sqlite + ]; + }; + }; + + # Tests the most basic user functionality expected from pdfding backup service + testScript = + { nodes, ... }: + let + inherit (nodes.machine.services.pdfding) port; + stateDir = "/var/lib/pdfding"; + in + # py + '' + # start vms + start_all() + + # create admin + machine.wait_for_unit("multi-user.target") + machine.succeed("DJANGO_SUPERUSER_PASSWORD=admin pdfding-manage createsuperuser --no-input --username admin --email admin@localhost") + + # login + endpoint = "http://localhost:${toString port}" + cookie_jar = "/tmp/cookies.txt" + machine.succeed(f""" + curl -f \ + -X POST -c {cookie_jar} -b {cookie_jar} \ + -d "csrfmiddlewaretoken=$(curl -f -c {cookie_jar} -s '{endpoint}/accountlogin/' | grep -oP 'name="csrfmiddlewaretoken" value="\\K[^"]+')" \ + -d "login=admin@localhost" \ + -d "password=admin" \ + {endpoint}/accountlogin/ + """) + + test_pdf = "${pkgs.pdfding.src}/pdfding/pdf/tests/data/dummy.pdf" + + # upload + machine.succeed(f""" + csrf_token=$(curl -f -b {cookie_jar} -c {cookie_jar} -s "{endpoint}/pdf/add" | grep -oP 'name="csrfmiddlewaretoken" value="\\K[^"]+') + curl -f \ + -c {cookie_jar} -b {cookie_jar} \ + -F "notes=" \ + -F "tag_string=" \ + -F "description=" \ + -F "use_file_name=on" \ + -F "name=test-upload" \ + -F "file=@{test_pdf};type=application/pdf" \ + -F "csrfmiddlewaretoken=$csrf_token" \ + -H "Referer: {endpoint}/pdf/add" \ + {endpoint}/pdf/add + """) + + # download + machine.succeed(f""" + pdf_id=$(curl -f -b {cookie_jar} -s "{endpoint}/pdf/" | grep -oP 'href="/pdf/view/\\K[^"]+' | head -1) + curl -f -b {cookie_jar} -o /tmp/downloaded.pdf "{endpoint}/pdf/download/$pdf_id" + """) + + # verify pdf in user's dir + machine.succeed("test -f ${stateDir}/media/1/pdf/*.pdf") + + # verify one entry exists in sqlite db + machine.succeed("sqlite3 ${stateDir}/db/db.sqlite3 'SELECT COUNT(*) FROM pdf_pdf' | grep -q '^1$'") + + machine.succeed(""" + source ${pdfding-s3-keys} + mc alias set garage --api S3v4 \ + http://[::]:3900 $BACKUP_ACCESS_KEY $BACKUP_SECRET_KEY + """) + + print(machine.succeed("realpath /run/current-system/sw/bin/backup-immediate")) + print(machine.succeed("backup-immediate")) + + # verify the backup s3 service has that pdf file + machine.wait_until_succeeds("mc stat garage/pdfding-bucket/1/pdf/dummy.pdf", timeout=10) + print(machine.succeed("mc stat garage/pdfding-bucket/1/pdf/dummy.pdf")) + ''; + + # Debug interactively with: + # - nix run .#nixosTests.pdfding.s3.driverInteractive -L + # - start_all() / run_tests() + interactive.sshBackdoor.enable = true; # ssh -o User=root vsock%3 + interactive.nodes.machine = + { config, ... }: + let + ports = [ + config.services.pdfding.port + 3901 + 3902 + ]; + in + { + # not needed, only for manual interactive debugging + virtualisation.memorySize = 4096; + environment.systemPackages = with pkgs; [ + htop + ]; + + virtualisation.forwardPorts = map (port: { + from = "host"; + host.port = port; + guest.port = port; + }) ports; + + # forwarded ports need to be accessible + networking.firewall.allowedTCPPorts = ports; + }; + + meta.maintainers = lib.teams.ngi.members; +} diff --git a/pkgs/by-name/pd/pdfding/package.nix b/pkgs/by-name/pd/pdfding/package.nix index 7e7715851396..6b07c84a90a7 100644 --- a/pkgs/by-name/pd/pdfding/package.nix +++ b/pkgs/by-name/pd/pdfding/package.nix @@ -5,6 +5,7 @@ fetchFromGitHub, fetchpatch2, makeWrapper, + nixosTests, }: let python = python3; @@ -197,6 +198,7 @@ python.pkgs.buildPythonPackage (finalAttrs: { passthru = { inherit python; + tests = nixosTests.pdfding; frontend = callPackage ./frontend.nix { }; updateScript = ./update.sh; }; From c2c711dd7ff0af56c0050c8d1cbf03dd6a48aba7 Mon Sep 17 00:00:00 2001 From: phanirithvij Date: Mon, 2 Feb 2026 15:10:48 +0530 Subject: [PATCH 3/3] pdfding: 1.4.1 -> 1.5.1 Signed-off-by: phanirithvij --- nixos/tests/web-apps/pdfding/basic.nix | 3 +- nixos/tests/web-apps/pdfding/postgres.nix | 4 +- nixos/tests/web-apps/pdfding/s3-backups.nix | 7 +-- pkgs/by-name/pd/pdfding/frontend.nix | 30 +++++------- pkgs/by-name/pd/pdfding/package.nix | 54 +++++++++------------ 5 files changed, 42 insertions(+), 56 deletions(-) diff --git a/nixos/tests/web-apps/pdfding/basic.nix b/nixos/tests/web-apps/pdfding/basic.nix index 9b1db591dfb5..d2a1df8a5d7e 100644 --- a/nixos/tests/web-apps/pdfding/basic.nix +++ b/nixos/tests/web-apps/pdfding/basic.nix @@ -85,6 +85,7 @@ -F "notes=" \ -F "tag_string=" \ -F "description=" \ + -F "collection=1" \ -F "use_file_name=on" \ -F "name=test-upload" \ -F "file=@{test_pdf};type=application/pdf" \ @@ -100,7 +101,7 @@ """) # verify pdf in user's dir - machine.succeed("test -f ${stateDir}/media/1/pdf/*.pdf") + machine.succeed("test -f ${stateDir}/media/1/default/pdf/*.pdf") # verify one entry exists in sqlite db machine.succeed("sqlite3 ${stateDir}/db/db.sqlite3 'SELECT COUNT(*) FROM pdf_pdf' | grep -q '^1$'") diff --git a/nixos/tests/web-apps/pdfding/postgres.nix b/nixos/tests/web-apps/pdfding/postgres.nix index 12a079402cb7..0fb9c174dc1a 100644 --- a/nixos/tests/web-apps/pdfding/postgres.nix +++ b/nixos/tests/web-apps/pdfding/postgres.nix @@ -60,8 +60,8 @@ print(machine.succeed("consume-immediate")) # verify pdf is in user's dir, and removed from consume dir - machine.wait_for_file("${stateDir}/media/1/pdf/dummy.pdf") - machine.fail("test -f ${stateDir}/consume/1/pdf/dummy.pdf") + machine.wait_for_file("${stateDir}/media/1/default/pdf/dummy.pdf") + machine.fail("test -f ${stateDir}/consume/1/default/pdf/dummy.pdf") # verify pdf is also in postgres db machine.succeed("sudo -u pdfding psql -tAc 'SELECT COUNT(*) FROM pdf_pdf' | grep -q '^1$'") diff --git a/nixos/tests/web-apps/pdfding/s3-backups.nix b/nixos/tests/web-apps/pdfding/s3-backups.nix index b8214eb66b9a..1339ab555114 100644 --- a/nixos/tests/web-apps/pdfding/s3-backups.nix +++ b/nixos/tests/web-apps/pdfding/s3-backups.nix @@ -133,6 +133,7 @@ in -F "notes=" \ -F "tag_string=" \ -F "description=" \ + -F "collection=1" \ -F "use_file_name=on" \ -F "name=test-upload" \ -F "file=@{test_pdf};type=application/pdf" \ @@ -148,7 +149,7 @@ in """) # verify pdf in user's dir - machine.succeed("test -f ${stateDir}/media/1/pdf/*.pdf") + machine.succeed("test -f ${stateDir}/media/1/default/pdf/*.pdf") # verify one entry exists in sqlite db machine.succeed("sqlite3 ${stateDir}/db/db.sqlite3 'SELECT COUNT(*) FROM pdf_pdf' | grep -q '^1$'") @@ -163,8 +164,8 @@ in print(machine.succeed("backup-immediate")) # verify the backup s3 service has that pdf file - machine.wait_until_succeeds("mc stat garage/pdfding-bucket/1/pdf/dummy.pdf", timeout=10) - print(machine.succeed("mc stat garage/pdfding-bucket/1/pdf/dummy.pdf")) + machine.wait_until_succeeds("mc stat garage/pdfding-bucket/1/default/pdf/dummy.pdf", timeout=10) + print(machine.succeed("mc stat garage/pdfding-bucket/1/default/pdf/dummy.pdf")) ''; # Debug interactively with: diff --git a/pkgs/by-name/pd/pdfding/frontend.nix b/pkgs/by-name/pd/pdfding/frontend.nix index 753db30af5e6..c408df8cdf9f 100644 --- a/pkgs/by-name/pd/pdfding/frontend.nix +++ b/pkgs/by-name/pd/pdfding/frontend.nix @@ -1,16 +1,17 @@ { stdenv, - nodejs, - npmHooks, fetchpatch2, fetchNpmDeps, fetchzip, fetchFromGitHub, + npmHooks, + tailwindcss_4, + nodejs, }: let - pdfjsVersion = "5.4.296"; # see update script - pdfjsHash = "sha256-b4W7wETq2CIZm2rJCmXEYvPhQtCbXS76L7GDvng6wn4="; + pdfjsVersion = "5.4.394"; # see update script + pdfjsHash = "sha256-pd7xwfvR9U1bHT5eblszYU3YJQwQwhuyDDiNj+fnyaQ="; pdfjs = fetchzip { url = "https://github.com/mozilla/pdf.js/releases/download/v${pdfjsVersion}/pdfjs-${pdfjsVersion}-dist.zip"; hash = pdfjsHash; @@ -27,29 +28,20 @@ let in stdenv.mkDerivation (finalAttrs: { pname = "pdfding-frontend"; - version = "1.4.1"; + version = "1.5.1"; src = fetchFromGitHub { owner = "mrmn2"; repo = "PdfDing"; tag = "v${finalAttrs.version}"; - hash = "sha256-8e80gMdB6U3977dIU7bIAAEguYmi+AWQgUgYPDLCYLI="; + hash = "sha256-PXkD+2k8/LmMWzZAj8qEK4mLoOKS4mDWcqe8AgoCdBU="; }; npmDeps = fetchNpmDeps { inherit (finalAttrs) src; name = "pdfding-frontend-${finalAttrs.version}-npm-deps"; - hash = "sha256-v1NFqDnFcRK8sd0bV3ck+LLMYQ90Dl1R1OnBTwWUVUg="; + hash = "sha256-SgL8QhRGONGhJBu6b8HSVqZPzJ+NojhVClBEH5ajCcc="; }; - patches = [ - # remove patch in 1.4.2 - # package.json has missing version and name - (fetchpatch2 { - url = "https://github.com/mrmn2/PdfDing/pull/203.patch?full_index=1"; - hash = "sha256-lKtpqKdyoGZdU4fTegto+YUIduIWbM82RQU9459NpC0="; - }) - ]; - nativeBuildInputs = [ nodejs npmHooks.npmConfigHook @@ -67,8 +59,10 @@ stdenv.mkDerivation (finalAttrs: { tailwindcss -i $out/pdfding/static/css/input.css -o $out/pdfding/static/css/tailwind.css --minify rm $out/pdfding/static/css/input.css - for i in build/pdf.mjs build/pdf.sandbox.mjs build/pdf.worker.mjs web/viewer.mjs; \ - do node_modules/terser/bin/terser $out/pdfding/static/pdfjs/$i --compress -o $out/pdfding/static/pdfjs/$i; done + for i in build/pdf.mjs build/pdf.sandbox.mjs build/pdf.worker.mjs web/viewer.mjs; + do + node_modules/terser/bin/terser $out/pdfding/static/pdfjs/$i --compress -o $out/pdfding/static/pdfjs/$i; + done npm run build diff --git a/pkgs/by-name/pd/pdfding/package.nix b/pkgs/by-name/pd/pdfding/package.nix index 6b07c84a90a7..30099f6978b0 100644 --- a/pkgs/by-name/pd/pdfding/package.nix +++ b/pkgs/by-name/pd/pdfding/package.nix @@ -1,49 +1,33 @@ { lib, - python3, callPackage, fetchFromGitHub, fetchpatch2, makeWrapper, nixosTests, + + python3, }: let python = python3; in python.pkgs.buildPythonPackage (finalAttrs: { pname = "pdfding"; - version = "1.4.1"; + version = "1.5.1"; src = fetchFromGitHub { owner = "mrmn2"; repo = "PdfDing"; tag = "v${finalAttrs.version}"; - hash = "sha256-rrUaqxDO16NAOic74jeYgN+7Alvo+fIIacJdSOg0hFM="; - # remove in 1.5.0 - postFetch = "mv $out/{license.txt,LICENSE}"; + hash = "sha256-PXkD+2k8/LmMWzZAj8qEK4mLoOKS4mDWcqe8AgoCdBU="; }; pyproject = true; patches = [ - # remove all patches in 1.5.0 (next version after 1.4.1) - # patch to add data_dir + # fixes two tests, remove patch in the next version + # https://github.com/mrmn2/PdfDing/pull/248 (fetchpatch2 { - url = "https://github.com/mrmn2/PdfDing/commit/f4945f2836ca8d972fcee2f00ef1d9cf217bada1.patch?full_index=1"; - hash = "sha256-VGjyIAVi+qd2WZ8FVKKC2ijLinoflO7RmPwIW1/oGcY="; - }) - # pyproject.toml still has 0.1.1 very old version - (fetchpatch2 { - url = "https://github.com/mrmn2/PdfDing/pull/203.patch?full_index=1"; - hash = "sha256-lKtpqKdyoGZdU4fTegto+YUIduIWbM82RQU9459NpC0="; - }) - # allows customising consume_schedule crontab - (fetchpatch2 { - url = "https://github.com/mrmn2/PdfDing/commit/96a13574718e0d27240eee8893fb799a02f24c05.patch?full_index=1"; - hash = "sha256-Stq392rIbsphvaE23GgFWb91KzpD6aOQu9MGDDoaO7s="; - }) - # allow specifying region for s3 backups - (fetchpatch2 { - url = "https://github.com/mrmn2/PdfDing/commit/3e412654f62d83b745111bd1d3587aca1a7739e1.patch?full_index=1"; - hash = "sha256-RQS3yJjrIaViFlm6it6zyRZOn+nTQE8qr8OpY+zYSCY="; + url = "https://github.com/mrmn2/PdfDing/pull/248/commits/8f6900dddd1dbbe1a1024a484f63b792dd022f99.patch?full_index=1"; + hash = "sha256-5oUC2TKL4X5IFy/41qViaafyUr4+bLBIovk9AWQmxZc="; }) ]; @@ -63,9 +47,12 @@ python.pkgs.buildPythonPackage (finalAttrs: { django-cleanup django-htmx gunicorn + huey markdown minio nh3 + oauthlib + pillow psycopg2-binary pypdf pypdfium2 @@ -74,14 +61,11 @@ python.pkgs.buildPythonPackage (finalAttrs: { rapidfuzz ruamel-yaml whitenoise - huey - pillow - oauthlib # dependecies required for django collectstatic - requests - pyjwt cryptography + pyjwt + requests ] ++ qrcode.optional-dependencies.pil ++ django-allauth.optional-dependencies.socialaccount; @@ -155,10 +139,9 @@ python.pkgs.buildPythonPackage (finalAttrs: { pythonRelaxDeps = [ "django" "django-allauth" - "huey" - "minio" + "django-htmx" "pypdf" - "pypdfium2" + "ruamel-yaml" ]; nativeCheckInputs = with python.pkgs; [ @@ -171,6 +154,12 @@ python.pkgs.buildPythonPackage (finalAttrs: { "--ignore=e2e" ]; + disabledTests = [ + # broken tests in 1.5.0 + "test_adjust_file_paths_to_ws_collection" + "test_oidc_callback" # AssertionError: 200 != 401 + ]; + /* fix two breaking tests by providing full out path AssertionError: Calls not found @@ -211,6 +200,7 @@ python.pkgs.buildPythonPackage (finalAttrs: { license = lib.licenses.agpl3Only; mainProgram = "pdfding-manage"; maintainers = with lib.maintainers; [ phanirithvij ]; + platforms = lib.platforms.unix; teams = with lib.teams; [ ngi ]; }; })