nixos/pdfding: init (#481927)

nixos/doc/manual/release-notes/rl-2605.section.md

@@ -18,6 +18,8 @@
 
 - [ImmichFrame](https://immichframe.dev/), display your photos from Immich as a digital photo frame. Available as `services.immichframe`.
 
+- [PdfDing](https://www.pdfding.com/), manage, view and edit your PDFs seamlessly on all your devices wherever you are. Available as [services.pdfding](#opt-services.pdfding.enable).
+
 - [reaction](https://reaction.ppom.me/), a daemon that scans program outputs for repeated patterns, and takes action. A common usage is to scan ssh and webserver logs, and to ban hosts that cause multiple authentication errors. A modern alternative to fail2ban. Available as [services.reaction](#opt-services.reaction.enable).
 
 - [rqbit](https://github.com/ikatson/rqbit), a bittorrent client written in Rust. It has HTTP API and Web UI, and can be used as a library. Available as [services.rqbit](#opt-services.rqbit.enable).

nixos/modules/module-list.nix

@@ -1714,6 +1714,7 @@
   ./services/web-apps/outline.nix
   ./services/web-apps/pairdrop.nix
   ./services/web-apps/part-db.nix
+  ./services/web-apps/pdfding.nix
   ./services/web-apps/peering-manager.nix
   ./services/web-apps/peertube-runner.nix
   ./services/web-apps/peertube.nix

nixos/modules/services/web-apps/pdfding.nix

@@ -0,0 +1,451 @@
+{
+  config,
+  lib,
+  pkgs,
+  ...
+}:
+let
+  inherit (lib)
+    concatStringsSep
+    mapAttrsToList
+    mkEnableOption
+    mkPackageOption
+    mkOption
+    optionalAttrs
+    optionalString
+    types
+    ;
+
+  cfg = config.services.pdfding;
+
+  stateDir = "/var/lib/pdfding";
+
+  usePostgres = cfg.database.type == "postgres";
+
+  envVars = {
+    # HOST_IP is used in the package derivation
+    HOST_IP = cfg.hostName;
+    HOST_NAME = concatStringsSep "," cfg.allowedHosts;
+    HOST_PORT = toString cfg.port;
+    DATABASE_TYPE = "";
+    DATA_DIR = stateDir;
+  }
+  // optionalAttrs usePostgres {
+    DATABASE_TYPE = "POSTGRES";
+    POSTGRES_PORT = toString cfg.database.port;
+    # Django Uses the unix domain socket
+    # if host is set to empty see https://docs.djangoproject.com/en/6.0/ref/settings/#host
+    POSTGRES_HOST = lib.optionalString (!cfg.database.createLocally) cfg.database.host;
+    POSTGRES_NAME = cfg.database.name;
+    POSTGRES_USER = cfg.database.user;
+  }
+  // optionalAttrs cfg.consume.enable {
+    CONSUME_ENABLE = "TRUE";
+    CONSUME_SCHEDULE = cfg.consume.schedule;
+  }
+  // optionalAttrs cfg.backup.enable {
+    BACKUP_ENABLE = "TRUE";
+    BACKUP_ENDPOINT = cfg.backup.endpoint;
+    BACKUP_SCHEDULE = cfg.backup.schedule;
+  }
+  // cfg.extraEnvironment;
+
+  envFile = pkgs.writeText "pdfding.env" (
+    lib.pipe envVars [
+      (mapAttrsToList (name: value: "${name}=\"${toString value}\""))
+      (concatStringsSep "\n")
+    ]
+  );
+
+  loadCreds =
+    optionalString (usePostgres && !cfg.database.createLocally) ''
+      export POSTGRES_PASSWORD="$(<${cfg.database.passwordFile})"
+    ''
+    + ''
+      export SECRET_KEY="$(<${cfg.secretKeyFile})"
+    '';
+
+  secretRecommendation = "Consider using a secret managing scheme such as `agenix` or `sops-nix` to generate this file.";
+in
+{
+  options.services.pdfding = {
+    enable = mkEnableOption "PdfDing service" // {
+      description = ''
+        Whether to enable PdfDing service.
+
+        To use the pdfding-manage CLI, add your user to the pdfding group:
+          users.users.<youruser>.extraGroups = [ "pdfding" ];
+      '';
+    };
+
+    package = mkPackageOption pkgs "pdfding" { };
+
+    user = mkOption {
+      type = types.str;
+      default = "pdfding";
+      description = "User account under which PdfDing runs";
+    };
+
+    group = mkOption {
+      type = types.str;
+      default = "pdfding";
+      description = "Group under which PdfDing runs";
+    };
+
+    hostName = mkOption {
+      type = types.str;
+      default = "0.0.0.0";
+      example = "pdfding.example.com";
+      description = "Listen address for PdfDing";
+    };
+
+    port = mkOption {
+      type = types.port;
+      default = 8000;
+      description = "Port on which PdfDing listens";
+    };
+
+    allowedHosts = mkOption {
+      type = types.listOf types.str;
+      default = [
+        "127.0.0.1"
+        "localhost"
+      ];
+      description = "Domains where PdfDing is allowed to run";
+    };
+
+    gunicorn.extraArgs = mkOption {
+      type = types.listOf types.str;
+      description = "Command line arguments passed to Gunicorn server.";
+      default = [ ];
+    };
+
+    extraEnvironment = mkOption {
+      type = types.attrsOf types.str;
+      default = { };
+      description = "Additional environment variables";
+    };
+
+    envFiles = mkOption {
+      type = types.listOf types.path;
+      description = "Environment variable files";
+      default = [ ];
+    };
+
+    secretKeyFile = mkOption {
+      type = types.path;
+      default = null;
+      description = "File containing the Django SECRET_KEY. ${secretRecommendation}";
+      example = "/run/secrets/pdfding-secret-key";
+    };
+
+    database = {
+      type = mkOption {
+        type = types.enum [
+          "sqlite"
+          "postgres"
+        ];
+        default = "sqlite";
+        description = "Database type to use";
+      };
+
+      host = mkOption {
+        type = types.str;
+        default = "";
+        description = "PostgreSQL host";
+      };
+
+      port = mkOption {
+        type = types.port;
+        default = 5432;
+        description = "PostgreSQL port";
+      };
+
+      name = mkOption {
+        type = types.str;
+        default = "pdfding";
+        description = "PostgreSQL database name";
+      };
+
+      user = mkOption {
+        type = types.str;
+        default = "pdfding";
+        description = "PostgreSQL user";
+      };
+
+      passwordFile = mkOption {
+        type = types.nullOr types.path;
+        default = null;
+        description = "File containing POSTGRES_PASSWORD. ${secretRecommendation}";
+        example = "/run/secrets/pdfding-db-password";
+      };
+
+      createLocally = mkOption {
+        type = types.bool;
+        default = false;
+        description = "Whether to create a local PostgreSQL database automatically";
+      };
+    };
+
+    consume = {
+      enable = mkEnableOption "Consume functionality" // {
+        description = ''
+          Bulk PDF import from consume directory.
+
+          When enabled, administrators can create per-user directories like /var/lib/pdfding/consume/<user_id>
+          with permissions allowing the pdfding user to read and write.
+          PDFs placed in these directories are automatically imported into user accounts.
+
+          PDFs are imported periodically via cronjob and successfully imported files
+          are automatically deleted from the consume directory.
+        '';
+      };
+      schedule = mkOption {
+        type = types.str;
+        default = "*/5 * * * *";
+        description = ''
+          The cron schedule for the consume task to trigger.
+          The format is "minute hour day month day_of_week"
+          Read
+            - https://github.com/mrmn2/PdfDing/blob/d0f21ec2f9fbee4b1a2f6b7e0e6c7ea7784ab1bc/pdfding/base/task_helpers.py#L5
+            - https://huey.readthedocs.io/en/latest/api.html#crontab
+        '';
+      };
+    };
+
+    backup = {
+      enable = mkEnableOption "Backup functionality" // {
+        description = ''
+          Automatic backup of important data to a AWS S3 (or compatible) instance.
+
+          When enabled and properly configured via environment variables,
+          important data is periodically uploaded to the specified s3
+          instance via cronjob.
+        '';
+      };
+      schedule = mkOption {
+        type = types.str;
+        default = "0 2 * * *";
+        description = ''
+          The cron schedule for the consume task to trigger.
+          The format is "minute hour day month day_of_week"
+          Read
+            - https://github.com/mrmn2/PdfDing/blob/d0f21ec2f9fbee4b1a2f6b7e0e6c7ea7784ab1bc/pdfding/base/task_helpers.py#L5
+            - https://huey.readthedocs.io/en/latest/api.html#crontab
+        '';
+      };
+      endpoint = mkOption {
+        type = types.nullOr types.str;
+        default = null;
+        description = "The s3 endpoint for backups";
+        example = "127.0.0.1:9000";
+      };
+    };
+
+    openFirewall = lib.mkOption {
+      type = types.bool;
+      default = false;
+      description = "Open ports in the firewall for the PdfDing web interface.";
+    };
+
+    installTestHelpers = mkOption {
+      type = types.bool;
+      default = false;
+      internal = true;
+      description = "Adds a few helper commands to systemPackages for nixos tests";
+    };
+  };
+
+  config = lib.mkIf cfg.enable {
+    assertions = [
+      {
+        assertion = cfg.secretKeyFile != null;
+        message = "services.pdfding.secretKeyFile must be set when using PdfDing";
+      }
+      {
+        assertion = cfg.backup.enable -> envVars.BACKUP_ENDPOINT != null;
+        message = "services.pdfding.extraEnvironment.BACKUP_ENDPOINT must be set when backup is enabled";
+      }
+      {
+        assertion = cfg.database.createLocally -> usePostgres;
+        message = "services.pdfding.database.createLocally is enabled but not database.type is not postgres";
+      }
+      {
+        assertion = cfg.database.createLocally -> cfg.database.host == "";
+        message = "services.pdfding.database.host must be empty when services.pdfding.database.createLocally is enabled";
+      }
+      {
+        assertion = cfg.database.createLocally -> cfg.database.passwordFile == null;
+        message = "specifying services.pdfding.database.passwordFile is not supported when used along with a local db setup";
+      }
+      {
+        assertion =
+          cfg.database.createLocally
+          -> cfg.database.user == cfg.user && cfg.database.user == cfg.database.name;
+        message = "services.pdfding.database.user should be the same as services.pdfding.user as well as services.pdfding.database.name when running a local db setup";
+      }
+    ];
+
+    networking.firewall = lib.mkIf cfg.openFirewall {
+      allowedTCPPorts = [ cfg.port ];
+    };
+
+    environment.systemPackages =
+      let
+        genWrapper =
+          name: cmd:
+          pkgs.writeShellScriptBin name ''
+            set -eou pipefail
+            set -a
+            ${lib.toShellVars cfg.extraEnvironment}
+            ${lib.concatMapStringsSep "\n" (f: "source ${f}") cfg.envFiles}
+            set +a
+            ${loadCreds}
+            sudo=exec
+            if [[ "$USER" != ${cfg.user} ]]; then
+              sudo='${config.security.wrapperDir}/sudo -E -u ${cfg.user}'
+            fi
+            ${cmd}
+          '';
+        commands.pdfding-manage = ''
+          $sudo ${lib.getExe cfg.package} "$@"
+        '';
+        commands.consume-immediate = ''
+          echo "from pdf.tasks import consume_function; consume_function(True)" | \
+            $sudo ${lib.getExe cfg.package} shell
+        '';
+        commands.backup-immediate = ''
+          echo "from backup.tasks import backup_function; backup_function()" | \
+            $sudo ${lib.getExe cfg.package} shell
+        '';
+        packages = lib.genAttrs (lib.attrNames commands) (name: genWrapper name commands.${name});
+      in
+      lib.mkMerge [
+        [
+          packages.pdfding-manage
+        ]
+        (lib.mkIf cfg.installTestHelpers [
+          packages.consume-immediate
+          packages.backup-immediate
+        ])
+      ];
+
+    users.users.${cfg.user} = {
+      isSystemUser = true;
+      group = cfg.group;
+    };
+
+    users.groups.${cfg.group} = { };
+
+    services.pdfding.envFiles = [ envFile ];
+
+    services.pdfding.extraEnvironment = {
+      DEFAULT_THEME = "dark";
+      DEFAULT_THEME_COLOR = "green";
+    };
+
+    services.pdfding.gunicorn.extraArgs = [
+      "--workers=4"
+      "--max-requests=1200"
+      "--max-requests-jitter=50"
+      "--log-level=error"
+    ];
+
+    systemd.services.pdfding = {
+      description = "PdfDing Web Service";
+      after = [
+        "network.target"
+      ]
+      ++ lib.optionals (usePostgres && cfg.database.createLocally) [ "postgresql.target" ];
+      wantedBy = [ "multi-user.target" ];
+
+      preStart = ''
+        ${loadCreds}
+        ${optionalString (usePostgres && cfg.database.createLocally)
+          # bash
+          ''
+            count=0
+            timeout=30
+            until ${pkgs.postgresql}/bin/pg_isready -p ${toString cfg.database.port}; do
+              if [ $count -ge $timeout ]; then
+                echo "Timed out waiting for PostgreSQL after $timeout seconds."
+                exit 1
+              fi
+              echo "Waiting for PostgreSQL... ($count/$timeout)"
+              sleep 1
+              count=$((count+1))
+            done
+          ''
+        }
+
+        ${cfg.package}/bin/pdfding-manage migrate
+        ${cfg.package}/bin/pdfding-manage clean_up
+      '';
+
+      script = ''
+        ${loadCreds}
+        exec ${cfg.package}/bin/pdfding-start ${toString cfg.gunicorn.extraArgs}
+      '';
+
+      serviceConfig = {
+        Type = "exec";
+        User = cfg.user;
+        Group = cfg.group;
+        EnvironmentFile = cfg.envFiles;
+        StateDirectory = [
+          "pdfding"
+          "pdfding/db"
+          "pdfding/media"
+        ]
+        ++ lib.optional cfg.consume.enable "pdfding/consume";
+        NoNewPrivileges = true;
+        PrivateTmp = true;
+        PrivateDevices = true;
+        ProtectSystem = "strict";
+        ProtectHome = true;
+        Restart = "on-failure";
+        RestartSec = "5s";
+      };
+    };
+
+    systemd.services.pdfding-background = lib.mkIf (cfg.consume.enable || cfg.backup.enable) {
+      description = "PdfDing Background Tasks (Huey)";
+      after = [ "pdfding.service" ];
+      wantedBy = [ "multi-user.target" ];
+      script = ''
+        ${loadCreds}
+        exec ${cfg.package}/bin/pdfding-manage run_huey
+      '';
+      serviceConfig = {
+        Type = "exec";
+        User = cfg.user;
+        Group = cfg.group;
+        WorkingDirectory = stateDir;
+        EnvironmentFile = cfg.envFiles;
+
+        NoNewPrivileges = true;
+        PrivateTmp = true;
+        PrivateDevices = true;
+        ProtectSystem = "strict";
+        ProtectHome = true;
+        ReadWritePaths = [ stateDir ];
+        Restart = "on-failure";
+        RestartSec = "5s";
+        TimeoutStopSec = 30;
+      };
+    };
+
+    services.postgresql = lib.mkIf cfg.database.createLocally {
+      enable = true;
+      ensureDatabases = [ cfg.database.name ];
+      ensureUsers = [
+        {
+          name = cfg.database.user;
+          ensureDBOwnership = true;
+        }
+      ];
+    };
+  };
+
+  meta.maintainers = lib.teams.ngi.members;
+}

nixos/tests/all-tests.nix

@@ -1231,6 +1231,10 @@ in
   pass-secret-service = runTest ./pass-secret-service.nix;
   password-option-override-ordering = runTest ./password-option-override-ordering.nix;
   patroni = handleTestOn [ "x86_64-linux" ] ./patroni.nix { };
+  pdfding = import ./web-apps/pdfding {
+    inherit (pkgs) lib;
+    inherit runTest;
+  };
   pdns-recursor = runTest ./pdns-recursor.nix;
   peerflix = runTest ./peerflix.nix;
   peering-manager = runTest ./web-apps/peering-manager.nix;

nixos/tests/web-apps/pdfding/basic.nix

@@ -0,0 +1,166 @@
+{
+  lib,
+  pkgs,
+  ...
+}:
+{
+  name = "PdfDing sqlite";
+
+  nodes = {
+    machine =
+      { ... }:
+      {
+        # WARNING: Do not add secrets to the world-readable /nix/store in a production deployment
+        # Use a secret management scheme instead https://wiki.nixos.org/wiki/Comparison_of_secret_managing_schemes
+        services.pdfding = {
+          enable = true;
+          secretKeyFile = pkgs.writeText "secretKeyFile" "test123";
+        };
+
+        environment.systemPackages = with pkgs; [
+          sqlite
+        ];
+
+        # test email validation works
+        services.pdfding.extraEnvironment = {
+          EMAIL_BACKEND = "SMTP";
+          SMTP_HOST = "localhost";
+          SMTP_PORT = "1025";
+          SMTP_USER = ""; # mailpit doesn't need auth
+          SMTP_PASSWORD = "";
+          SMTP_USE_TLS = "FALSE";
+          SMTP_USE_SSL = "FALSE";
+        };
+
+        # enable mailpit
+        services.mailpit.instances.default = { };
+      };
+  };
+
+  # Test the most basic user functionality expected from pdfding.
+  # Heavy e2e test suite is implemented in e2e.nix
+  testScript =
+    { nodes, ... }:
+    let
+      inherit (nodes.machine.services.pdfding) port;
+      mailpitApiEndpoint = "http://${nodes.machine.services.mailpit.instances.default.listen}/api/v1";
+      stateDir = "/var/lib/pdfding";
+    in
+    # py
+    ''
+      import json
+      from pprint import pprint
+
+      # start vms
+      start_all()
+
+      # create admin
+      machine.wait_for_unit("multi-user.target")
+      machine.succeed("DJANGO_SUPERUSER_PASSWORD=admin pdfding-manage createsuperuser --no-input --username admin --email admin@localhost")
+
+      cookie_jar = "/tmp/cookies.txt"
+      endpoint = "http://localhost:${toString port}"
+
+      with subtest("login and basic usage"):
+        # login
+        machine.succeed(f"""
+          curl -f \
+            -X POST -c {cookie_jar} -b {cookie_jar} \
+            -d "csrfmiddlewaretoken=$(curl -f -c {cookie_jar} -s '{endpoint}/accountlogin/' | grep -oP 'name="csrfmiddlewaretoken" value="\\K[^"]+')" \
+            -d "login=admin@localhost" \
+            -d "password=admin" \
+            {endpoint}/accountlogin/
+        """)
+
+        test_pdf = "${pkgs.pdfding.src}/pdfding/pdf/tests/data/dummy.pdf"
+
+        # verify no pdfs exist in db
+        machine.succeed("sqlite3 ${stateDir}/db/db.sqlite3 'SELECT COUNT(*) FROM pdf_pdf' | grep -q '^0$'")
+
+        # upload
+        machine.succeed(f"""
+          csrf_token=$(curl -f -b {cookie_jar} -c {cookie_jar} -s "{endpoint}/pdf/add" | grep -oP 'name="csrfmiddlewaretoken" value="\\K[^"]+')
+          curl -f \
+            -c {cookie_jar} -b {cookie_jar} \
+            -F "notes=" \
+            -F "tag_string=" \
+            -F "description=" \
+            -F "collection=1" \
+            -F "use_file_name=on" \
+            -F "name=test-upload" \
+            -F "file=@{test_pdf};type=application/pdf" \
+            -F "csrfmiddlewaretoken=$csrf_token" \
+            -H "Referer: {endpoint}/pdf/add" \
+            {endpoint}/pdf/add
+        """)
+
+        # download
+        machine.succeed(f"""
+          pdf_id=$(curl -f -b {cookie_jar} -s "{endpoint}/pdf/" | grep -oP 'href="/pdf/view/\\K[^"]+' | head -1)
+          curl -f -b {cookie_jar} -o /tmp/downloaded.pdf "{endpoint}/pdf/download/$pdf_id"
+        """)
+
+        # verify pdf in user's dir
+        machine.succeed("test -f ${stateDir}/media/1/default/pdf/*.pdf")
+
+        # verify one entry exists in sqlite db
+        machine.succeed("sqlite3 ${stateDir}/db/db.sqlite3 'SELECT COUNT(*) FROM pdf_pdf' | grep -q '^1$'")
+
+      with subtest("email validation"):
+        # check we can reach mailpit
+        machine.succeed("curl -f ${mailpitApiEndpoint}/info")
+
+        # check that no emails exist
+        result = json.loads(machine.succeed("curl -sf ${mailpitApiEndpoint}/messages"))
+        pprint(result)
+        assert result["total"] == 0
+
+        # signup
+        machine.succeed(f"""
+          curl -f \
+            -X POST -c {cookie_jar} -b {cookie_jar} \
+            -d "csrfmiddlewaretoken=$(curl -f -c {cookie_jar} -s '{endpoint}/accountsignup/' | grep -oP 'name="csrfmiddlewaretoken" value="\\K[^"]+')" \
+            -d "email=pdfding_new_user@example.com" \
+            -d "password1=foobarbaz" \
+            -d "password2=foobarbaz" \
+            {endpoint}/accountsignup/
+        """)
+
+        # wait a bit for email to be processed
+        machine.sleep(3)
+
+        # verify the email was received by mailpit
+        result = json.loads(machine.succeed("curl -s ${mailpitApiEndpoint}/messages"))
+        pprint(result)
+        assert result["total"] == 1
+        assert result["messages"][0]["To"][0]["Address"] == "pdfding_new_user@example.com"
+    '';
+
+  # Debug interactively with:
+  # - nix run .#nixosTests.pdfding.basic.driverInteractive -L
+  # - start_all() / run_tests()
+  interactive.sshBackdoor.enable = true; # ssh -o User=root vsock%3
+  interactive.nodes.machine =
+    { config, ... }:
+    let
+      port = config.services.pdfding.port;
+    in
+    {
+      # not needed, only for manual interactive debugging
+      virtualisation.memorySize = 4096;
+      environment.systemPackages = with pkgs; [
+        htop
+      ];
+
+      virtualisation.forwardPorts = map (port: {
+        from = "host";
+        host.port = port;
+        guest.port = port;
+      }) [ port ];
+
+      # forwarded ports need to be accessible
+      networking.firewall.allowedTCPPorts = [ port ];
+    };
+
+  meta.maintainers = lib.teams.ngi.members;
+}

nixos/tests/web-apps/pdfding/default.nix

@@ -0,0 +1,7 @@
+{ lib, runTest }:
+lib.recurseIntoAttrs {
+  basic = runTest ./basic.nix;
+  e2e = runTest ./e2e.nix;
+  postgres = runTest ./postgres.nix;
+  s3-backups = runTest ./s3-backups.nix;
+}

nixos/tests/web-apps/pdfding/e2e.nix

@@ -0,0 +1,81 @@
+{
+  lib,
+  pkgs,
+  ...
+}:
+{
+  name = "PdfDing e2e tests";
+
+  nodes = {
+    machine =
+      { ... }:
+      {
+        environment.systemPackages = [
+          (pkgs.writeShellScriptBin "tests_e2e" (
+            let
+              pythonPath =
+                with pkgs.python3Packages;
+                [
+                  playwright
+                  pkgs.pdfding
+                ]
+                ++ pkgs.pdfding.dependencies
+                ++ pkgs.pdfding.optional-dependencies.e2e;
+            in
+            # bash
+            ''
+              export PLAYWRIGHT_BROWSERS_PATH=${pkgs.playwright-driver.browsers}
+              export PYTHONPATH=${pkgs.python3Packages.makePythonPath pythonPath}
+              export PATH=${pkgs.pdfding.python}/bin:$PATH
+              export E2E_TESTS=1
+              cd $(mktemp -d)
+              cp -r --no-preserve=all ${pkgs.pdfding.src} source
+              cd source
+              cp -ru --no-preserve=all ${pkgs.pdfding.frontend}/pdfding/static pdfding
+              # some tests are flaky due to timeouts, re-run them
+              python -m pytest pdfding/e2e \
+                -x -r aR \
+                --reruns 10 --only-rerun TimeoutError
+            ''
+            # see https://github.com/MrBin99/django-vite/issues/95
+            # tdlr; collectstatic is not important for e2e tests which uses StaticLiveServerTestCase
+            # it only cares about files in static/
+          ))
+        ];
+      };
+  };
+
+  testScript =
+    { nodes, ... }:
+    # py
+    ''
+      # start
+      start_all()
+      machine.wait_for_unit("multi-user.target")
+
+      machine.succeed("tests_e2e | systemd-cat")
+    '';
+
+  # Debug interactively with:
+  # - nix run .#nixosTests.pdfding.e2e.driverInteractive -L
+  # - start_all() / run_tests()
+  interactive.sshBackdoor.enable = true; # ssh -o User=root vsock%3
+  interactive.nodes.machine =
+    { config, ... }:
+    {
+      imports = [
+        # enable graphical session + users (alice, bob)
+        ./common/x11.nix
+        ./common/user-account.nix
+      ];
+      services.xserver.enable = true;
+      test-support.displayManager.auto.user = "alice";
+      virtualisation.memorySize = lib.mkForce 8192;
+      environment.systemPackages = with pkgs; [
+        htop
+      ];
+      # env DISPLAY=:0 sudo -u alice tests_e2e | systemd-cat
+    };
+
+  meta.maintainers = lib.teams.ngi.members;
+}

nixos/tests/web-apps/pdfding/postgres.nix

@@ -0,0 +1,97 @@
+{
+  lib,
+  pkgs,
+  ...
+}:
+{
+  name = "PdfDing postgres";
+
+  nodes = {
+    machine =
+      { config, ... }:
+      {
+        # WARNING: Do not add secrets to the world-readable /nix/store in a production deployment
+        # Use a secret management scheme instead https://wiki.nixos.org/wiki/Comparison_of_secret_managing_schemes
+        services.pdfding = {
+          enable = true;
+          secretKeyFile = pkgs.writeText "secretKeyFile" "test123";
+          database.createLocally = true;
+          database.type = "postgres";
+          consume.enable = true; # allows bulk importing pdf files from the backend
+          consume.schedule = "*/1 * * * *"; # once every minute
+          installTestHelpers = true;
+        };
+
+        environment.systemPackages = [
+          config.services.postgresql.finalPackage
+        ];
+      };
+  };
+
+  # Tests the most basic user functionality expected from pdfding with postgres and consume feature
+  testScript =
+    { nodes, ... }:
+    let
+      stateDir = "/var/lib/pdfding";
+    in
+    # py
+    ''
+      # start
+      start_all()
+
+      # create admin
+      machine.wait_for_unit("multi-user.target")
+
+      machine.succeed("DJANGO_SUPERUSER_PASSWORD=admin pdfding-manage createsuperuser --no-input --username admin --email admin@localhost")
+
+      test_pdf = "${pkgs.pdfding.src}/pdfding/pdf/tests/data/dummy.pdf"
+
+      # copy to consume dir
+      machine.succeed(f"sudo -u pdfding bash -c 'mkdir -p ${stateDir}/consume/1 && cp {test_pdf} ${stateDir}/consume/1/'")
+
+      # check the file was copied
+      output = machine.succeed("sudo -u pdfding bash -c 'ls -l ${stateDir}/consume/1/'")
+      assert "dummy.pdf" in output, "dummy pdf file not found"
+
+      # check there are no pdfs
+      machine.succeed("sudo -u pdfding psql -tAc 'SELECT COUNT(*) FROM pdf_pdf' | grep -q '^0$'")
+
+      print(machine.succeed("realpath /run/current-system/sw/bin/consume-immediate"))
+      print(machine.succeed("consume-immediate"))
+
+      # verify pdf is in user's dir, and removed from consume dir
+      machine.wait_for_file("${stateDir}/media/1/default/pdf/dummy.pdf")
+      machine.fail("test -f ${stateDir}/consume/1/default/pdf/dummy.pdf")
+
+      # verify pdf is also in postgres db
+      machine.succeed("sudo -u pdfding psql -tAc 'SELECT COUNT(*) FROM pdf_pdf' | grep -q '^1$'")
+    '';
+
+  # Debug interactively with:
+  # - nix run .#nixosTests.pdfding.postgres.driverInteractive -L
+  # - start_all() / run_tests()
+  interactive.sshBackdoor.enable = true; # ssh -o User=root vsock%3
+  interactive.nodes.machine =
+    { config, ... }:
+    let
+      port = config.services.pdfding.port;
+    in
+    {
+      # not needed, only for manual interactive debugging
+      virtualisation.memorySize = 4096;
+      environment.systemPackages = with pkgs; [
+        htop
+      ];
+
+      virtualisation.forwardPorts = map (port: {
+        from = "host";
+        host.port = port;
+        guest.port = port;
+      }) [ port ];
+
+      # forwarded ports need to be accessible
+      networking.firewall.allowedTCPPorts = [ port ];
+    };
+
+  meta.maintainers = lib.teams.ngi.members;
+}

nixos/tests/web-apps/pdfding/s3-backups.nix

@@ -0,0 +1,202 @@
+{
+  lib,
+  pkgs,
+  ...
+}:
+let
+  # WARNING: Do not add secrets to the world-readable /nix/store in a production deployment
+  # Use a secret management scheme instead https://wiki.nixos.org/wiki/Comparison_of_secret_managing_schemes
+
+  # note: In a production deployment use `garage key generate`, along with the steps specified in the getting started guide of garage.
+  # (sops-nix or agenix for configuring secrets for garage is out of scope here)
+  garageAccessKey = "GK0a0a0a0b0b0b0c0c0c0d0d0d";
+  garageSecretKey = "0a0a0a0a0b0b0b0b0c0c0c0c0d0d0d0d1a1a1a1a1b1b1b1b1c1c1c1c1d1d1d1d";
+  pdfding-s3-keys = pkgs.writeText "pdfding-s3-keys" ''
+    BACKUP_ACCESS_KEY=${garageAccessKey}
+    BACKUP_SECRET_KEY=${garageSecretKey}
+  '';
+in
+{
+  name = "PdfDing s3 backups";
+
+  nodes = {
+    machine =
+      { config, ... }:
+      {
+        services.pdfding = {
+          enable = true;
+          secretKeyFile = pkgs.writeText "secretKeyFile" "test123";
+
+          backup.enable = true;
+          backup.schedule = "*/1 * * * *";
+          backup.endpoint = "[::]:3900";
+          extraEnvironment.BACKUP_BUCKET_NAME = "pdfding-bucket";
+          extraEnvironment.BACKUP_REGION = "garage";
+
+          envFiles = [ pdfding-s3-keys ];
+          installTestHelpers = true;
+        };
+
+        # Setup a local garage service for the backup feature
+        # taken from garage nixosTest
+        services.garage = {
+          enable = true;
+          package = pkgs.garage_2;
+          settings = {
+            rpc_bind_addr = "[::]:3901";
+            rpc_public_addr = "[::1]:3901";
+            rpc_secret = "5c1915fa04d0b6739675c61bf5907eb0fe3d9c69850c83820f51b4d25d13868c";
+
+            s3_api = {
+              s3_region = "garage";
+              api_bind_addr = "[::]:3900";
+              root_domain = ".s3.garage";
+            };
+
+            s3_web = {
+              bind_addr = "[::]:3902";
+              root_domain = ".web.garage";
+              index = "index.html";
+            };
+
+            replication_factor = 1;
+          };
+        };
+
+        # setup garage bucket and credentials
+        # note: The nixos module has no option to specify secrets declaratively
+        systemd.services.garage.postStart = ''
+          export PATH="$PATH:${config.services.garage.package}/bin"
+
+          # wait for garage to be up
+          while ! garage status >/dev/null 2>&1; do sleep 2; done
+
+          if ! garage bucket list | grep -q pdfding-bucket; then
+            garage layout assign -z dc1 -c 1G $(garage status | cut -d' ' -f1 | tail -1)
+            garage layout apply --version 1
+
+            # note: the key id should be valid (starts with `GK`, followed by 12 hex-encoded bytes)
+            #       the secret key should be valid (composed of 32 hex-encoded bytes)
+            garage key import ${garageAccessKey} ${garageSecretKey} -n pdfding-key --yes
+
+            garage bucket create pdfding-bucket
+
+            garage bucket allow --read --write --owner pdfding-bucket --key pdfding-key
+          fi
+        '';
+
+        # Garage requires at least 1GiB of free disk space to run.
+        virtualisation.diskSize = 2 * 1024;
+
+        environment.systemPackages = with pkgs; [
+          minio-client
+          sqlite
+        ];
+      };
+  };
+
+  # Tests the most basic user functionality expected from pdfding backup service
+  testScript =
+    { nodes, ... }:
+    let
+      inherit (nodes.machine.services.pdfding) port;
+      stateDir = "/var/lib/pdfding";
+    in
+    # py
+    ''
+      # start vms
+      start_all()
+
+      # create admin
+      machine.wait_for_unit("multi-user.target")
+      machine.succeed("DJANGO_SUPERUSER_PASSWORD=admin pdfding-manage createsuperuser --no-input --username admin --email admin@localhost")
+
+      # login
+      endpoint = "http://localhost:${toString port}"
+      cookie_jar = "/tmp/cookies.txt"
+      machine.succeed(f"""
+        curl -f \
+          -X POST -c {cookie_jar} -b {cookie_jar} \
+          -d "csrfmiddlewaretoken=$(curl -f -c {cookie_jar} -s '{endpoint}/accountlogin/' | grep -oP 'name="csrfmiddlewaretoken" value="\\K[^"]+')" \
+          -d "login=admin@localhost" \
+          -d "password=admin" \
+          {endpoint}/accountlogin/
+      """)
+
+      test_pdf = "${pkgs.pdfding.src}/pdfding/pdf/tests/data/dummy.pdf"
+
+      # upload
+      machine.succeed(f"""
+        csrf_token=$(curl -f -b {cookie_jar} -c {cookie_jar} -s "{endpoint}/pdf/add" | grep -oP 'name="csrfmiddlewaretoken" value="\\K[^"]+')
+        curl -f \
+          -c {cookie_jar} -b {cookie_jar} \
+          -F "notes=" \
+          -F "tag_string=" \
+          -F "description=" \
+          -F "collection=1" \
+          -F "use_file_name=on" \
+          -F "name=test-upload" \
+          -F "file=@{test_pdf};type=application/pdf" \
+          -F "csrfmiddlewaretoken=$csrf_token" \
+          -H "Referer: {endpoint}/pdf/add" \
+          {endpoint}/pdf/add
+      """)
+
+      # download
+      machine.succeed(f"""
+        pdf_id=$(curl -f -b {cookie_jar} -s "{endpoint}/pdf/" | grep -oP 'href="/pdf/view/\\K[^"]+' | head -1)
+        curl -f -b {cookie_jar} -o /tmp/downloaded.pdf "{endpoint}/pdf/download/$pdf_id"
+      """)
+
+      # verify pdf in user's dir
+      machine.succeed("test -f ${stateDir}/media/1/default/pdf/*.pdf")
+
+      # verify one entry exists in sqlite db
+      machine.succeed("sqlite3 ${stateDir}/db/db.sqlite3 'SELECT COUNT(*) FROM pdf_pdf' | grep -q '^1$'")
+
+      machine.succeed("""
+        source ${pdfding-s3-keys}
+        mc alias set garage --api S3v4 \
+          http://[::]:3900 $BACKUP_ACCESS_KEY $BACKUP_SECRET_KEY
+      """)
+
+      print(machine.succeed("realpath /run/current-system/sw/bin/backup-immediate"))
+      print(machine.succeed("backup-immediate"))
+
+      # verify the backup s3 service has that pdf file
+      machine.wait_until_succeeds("mc stat garage/pdfding-bucket/1/default/pdf/dummy.pdf", timeout=10)
+      print(machine.succeed("mc stat garage/pdfding-bucket/1/default/pdf/dummy.pdf"))
+    '';
+
+  # Debug interactively with:
+  # - nix run .#nixosTests.pdfding.s3.driverInteractive -L
+  # - start_all() / run_tests()
+  interactive.sshBackdoor.enable = true; # ssh -o User=root vsock%3
+  interactive.nodes.machine =
+    { config, ... }:
+    let
+      ports = [
+        config.services.pdfding.port
+        3901
+        3902
+      ];
+    in
+    {
+      # not needed, only for manual interactive debugging
+      virtualisation.memorySize = 4096;
+      environment.systemPackages = with pkgs; [
+        htop
+      ];
+
+      virtualisation.forwardPorts = map (port: {
+        from = "host";
+        host.port = port;
+        guest.port = port;
+      }) ports;
+
+      # forwarded ports need to be accessible
+      networking.firewall.allowedTCPPorts = ports;
+    };
+
+  meta.maintainers = lib.teams.ngi.members;
+}

pkgs/by-name/pd/pdfding/frontend.nix

@@ -0,0 +1,81 @@
+{
+  stdenv,
+  fetchpatch2,
+  fetchNpmDeps,
+  fetchzip,
+  fetchFromGitHub,
+  npmHooks,
+
+  tailwindcss_4,
+  nodejs,
+}:
+let
+  pdfjsVersion = "5.4.394"; # see update script
+  pdfjsHash = "sha256-pd7xwfvR9U1bHT5eblszYU3YJQwQwhuyDDiNj+fnyaQ=";
+  pdfjs = fetchzip {
+    url = "https://github.com/mozilla/pdf.js/releases/download/v${pdfjsVersion}/pdfjs-${pdfjsVersion}-dist.zip";
+    hash = pdfjsHash;
+    stripRoot = false;
+    postFetch = ''
+      rm -rf $out/web/locale \
+      $out/web/standard_fonts \
+      $out/web/compressed.tracemonkey-pldi-09.pdf
+
+      # remove source maps
+      find "$out" -name '*.map' -exec rm -f '{}' \;
+    '';
+  };
+in
+stdenv.mkDerivation (finalAttrs: {
+  pname = "pdfding-frontend";
+  version = "1.5.1";
+  src = fetchFromGitHub {
+    owner = "mrmn2";
+    repo = "PdfDing";
+    tag = "v${finalAttrs.version}";
+    hash = "sha256-PXkD+2k8/LmMWzZAj8qEK4mLoOKS4mDWcqe8AgoCdBU=";
+  };
+
+  npmDeps = fetchNpmDeps {
+    inherit (finalAttrs) src;
+    name = "pdfding-frontend-${finalAttrs.version}-npm-deps";
+    hash = "sha256-SgL8QhRGONGhJBu6b8HSVqZPzJ+NojhVClBEH5ajCcc=";
+  };
+
+  nativeBuildInputs = [
+    nodejs
+    npmHooks.npmConfigHook
+    # it is in package.json and thus node_modules but no cli executable
+    tailwindcss_4
+  ];
+
+  # keeping the file structure same as upstream to minimise confusion
+  buildPhase = ''
+    runHook preBuild
+    mkdir -p $out/pdfding
+    cp -r --no-preserve=mode pdfding/static $out/pdfding/static
+    cp -r --no-preserve=mode ${finalAttrs.passthru.pdfjs} $out/pdfding/static/pdfjs
+
+    tailwindcss -i $out/pdfding/static/css/input.css -o $out/pdfding/static/css/tailwind.css --minify
+    rm $out/pdfding/static/css/input.css
+
+    for i in build/pdf.mjs build/pdf.sandbox.mjs build/pdf.worker.mjs web/viewer.mjs;
+    do
+      node_modules/terser/bin/terser $out/pdfding/static/pdfjs/$i --compress -o $out/pdfding/static/pdfjs/$i;
+    done
+
+    npm run build
+
+    cp -r pdfding/static/js $out/pdfding/static
+
+    runHook postBuild
+  '';
+
+  passthru = {
+    inherit pdfjs;
+  };
+
+  meta = {
+    description = "PdfDing frontend";
+  };
+})

pkgs/by-name/pd/pdfding/package.nix

@@ -0,0 +1,206 @@
+{
+  lib,
+  callPackage,
+  fetchFromGitHub,
+  fetchpatch2,
+  makeWrapper,
+  nixosTests,
+
+  python3,
+}:
+let
+  python = python3;
+in
+python.pkgs.buildPythonPackage (finalAttrs: {
+  pname = "pdfding";
+  version = "1.5.1";
+  src = fetchFromGitHub {
+    owner = "mrmn2";
+    repo = "PdfDing";
+    tag = "v${finalAttrs.version}";
+    hash = "sha256-PXkD+2k8/LmMWzZAj8qEK4mLoOKS4mDWcqe8AgoCdBU=";
+  };
+  pyproject = true;
+
+  patches = [
+    # fixes two tests, remove patch in the next version
+    # https://github.com/mrmn2/PdfDing/pull/248
+    (fetchpatch2 {
+      url = "https://github.com/mrmn2/PdfDing/pull/248/commits/8f6900dddd1dbbe1a1024a484f63b792dd022f99.patch?full_index=1";
+      hash = "sha256-5oUC2TKL4X5IFy/41qViaafyUr4+bLBIovk9AWQmxZc=";
+    })
+  ];
+
+  # remove supervisor from dependencies
+  postPatch = ''
+    sed -i 's/supervisor.*$//' pyproject.toml
+
+    substituteInPlace pdfding/backup/tests/test_management.py pdfding/backup/tests/test_tasks.py \
+      --replace-fail "Path(__file__).parents[2]" "Path('$PDFDING_OUT_DIR')"
+  '';
+
+  dependencies =
+    with python.pkgs;
+    [
+      django
+      django-allauth
+      django-cleanup
+      django-htmx
+      gunicorn
+      huey
+      markdown
+      minio
+      nh3
+      oauthlib
+      pillow
+      psycopg2-binary
+      pypdf
+      pypdfium2
+      python-magic
+      qrcode
+      rapidfuzz
+      ruamel-yaml
+      whitenoise
+
+      # dependecies required for django collectstatic
+      cryptography
+      pyjwt
+      requests
+    ]
+    ++ qrcode.optional-dependencies.pil
+    ++ django-allauth.optional-dependencies.socialaccount;
+
+  build-system = with python.pkgs; [ poetry-core ];
+
+  nativeBuildInputs = [
+    makeWrapper
+  ];
+
+  optional-dependencies = {
+    e2e = with python.pkgs; [
+      pytest
+      pytest-django
+      pytest-playwright
+      pytest-rerunfailures # required to retry some flaky e2e tests
+    ];
+  };
+
+  preBuild = ''
+    # remove originals, copy from frontend
+    rm -rf pdfding/static
+    ln -s ${finalAttrs.passthru.frontend}/pdfding/static pdfding/static
+
+    # staticfiles step requires prod configuration, remove dev.py
+    mv pdfding/core/settings/dev.py dev.py.bak
+
+    ${python.pythonOnBuildForHost.interpreter} pdfding/manage.py collectstatic
+
+    # not needed, now we have staticfiles directory
+    rm -rf pdfding/static
+
+    # the following is from upstream's Dockerfile
+
+    # remove django md5 hash from filenames of pdfjs as it will mess up the relative imports because of the whitenoise setup
+    export PDFJS_PATH="pdfding/staticfiles/pdfjs"
+    for file_name in $(find $PDFJS_PATH -type f -not -path "$PDFJS_PATH/web/images/*")
+    do
+      if [[ $file_name =~ "LICENSE" ]]; then
+        new=$(echo "$file_name" | sed -E "s/LICENSE\.[a-zA-Z0-9]{12}/LICENSE/");
+      else
+        new=$(echo "$file_name" | sed -E "s/\.[a-zA-Z0-9]{12}\./\./");
+      fi;
+      mv -- "$file_name" "$new";
+    done \
+    && echo 'Successfully removed hash from pdfjs files'
+
+    echo "VERSION = '${finalAttrs.version}'" > pdfding/core/settings/version.py;
+  '';
+
+  env.PDFDING_OUT_DIR = "${placeholder "out"}/${python.sitePackages}/pdfding";
+
+  makeWrapperArgs = [
+    "--set-default DATA_DIR /var/lib/pdfding"
+    # allow for gunicorn processes to have access to Python packages
+    "--prefix PYTHONPATH : "
+    "${python.pkgs.makePythonPath finalAttrs.passthru.dependencies}:${finalAttrs.env.PDFDING_OUT_DIR}"
+  ];
+
+  postInstall = ''
+    mkdir -p $out/bin
+
+    makeWrapper "$PDFDING_OUT_DIR/manage.py" $out/bin/pdfding-manage \
+      $makeWrapperArgs
+
+    makeWrapper ${lib.getExe python.pkgs.gunicorn} $out/bin/pdfding-start \
+      --add-flags '--bind ''${HOST_IP:-127.0.0.1}:''${HOST_PORT:-8080} core.wsgi:application' \
+      $makeWrapperArgs
+  '';
+
+  pythonRelaxDeps = [
+    "django"
+    "django-allauth"
+    "django-htmx"
+    "pypdf"
+    "ruamel-yaml"
+  ];
+
+  nativeCheckInputs = with python.pkgs; [
+    pytest-django
+    pytestCheckHook
+  ];
+
+  # from .github/workflows/tests.yaml
+  pytestFlags = [
+    "--ignore=e2e"
+  ];
+
+  disabledTests = [
+    # broken tests in 1.5.0
+    "test_adjust_file_paths_to_ws_collection"
+    "test_oidc_callback" # AssertionError: 200 != 401
+  ];
+
+  /*
+    fix two breaking tests by providing full out path
+    AssertionError: Calls not found
+    AssertionError: 'add_file_to_minio' does not contain all of ...
+  */
+  preCheck = ''
+    # dev.py is required for tests, restore it
+    mv dev.py.bak $PDFDING_OUT_DIR/core/settings/dev.py
+
+    # tests should run in pdfding directory
+    pushd pdfding
+  '';
+
+  postCheck = ''
+    # come out of the pdfding directory
+    popd
+
+    # remove dev.py
+    rm $PDFDING_OUT_DIR/core/settings/dev.py
+  '';
+
+  pythonImportsCheck = [
+    "pdfding"
+  ];
+
+  passthru = {
+    inherit python;
+    tests = nixosTests.pdfding;
+    frontend = callPackage ./frontend.nix { };
+    updateScript = ./update.sh;
+  };
+
+  meta = {
+    changelog = "https://github.com/mrmn2/PdfDing/blob/${finalAttrs.src.rev}/CHANGELOG.md";
+    description = "Selfhosted PDF manager, viewer and editor offering a seamless user experience on multiple devices";
+    downloadPage = "https://github.com/mrmn2/PdfDing";
+    homepage = "https://pdfding.com";
+    license = lib.licenses.agpl3Only;
+    mainProgram = "pdfding-manage";
+    maintainers = with lib.maintainers; [ phanirithvij ];
+    platforms = lib.platforms.unix;
+    teams = with lib.teams; [ ngi ];
+  };
+})

pkgs/by-name/pd/pdfding/update.sh

@@ -0,0 +1,53 @@
+#!/usr/bin/env nix-shell
+#!nix-shell -i bash -p curl jq nix-update gitMinimal prefetch-npm-deps coreutils
+
+set -x
+set -eou pipefail
+
+version=$(curl ${GITHUB_TOKEN:+ -H "Authorization: Bearer $GITHUB_TOKEN"} -sL https://api.github.com/repos/mrmn2/PdfDing/releases/latest | jq -r '.tag_name')
+
+if [[ "v${UPDATE_NIX_OLD_VERSION:-}" == "$version" ]]; then
+  echo "Already up-to-date, version: $version"
+  exit 0
+fi
+
+# source hashes
+nix-update --version="$version" pdfding
+nix-update --version="$version" pdfding.frontend
+
+NIXPKGS_PATH="$(git rev-parse --show-toplevel)"
+PACKAGE_DIR="$( cd -- "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )"
+
+TMPDIR="$(mktemp -d)"
+trap 'rm -rf "$TMPDIR"' EXIT
+cd "$TMPDIR"
+
+src="$(nix-build --no-link "$NIXPKGS_PATH" -A pdfding.src)"
+cp "$src"/{package.json,package-lock.json} .
+
+# npmDeps hash
+prev_npm_hash="$(
+  nix-instantiate "$NIXPKGS_PATH" \
+    --eval --json \
+    -A pdfding.frontend.npmDeps.hash |
+    jq -r .
+)"
+new_npm_hash="$(prefetch-npm-deps ./package-lock.json)"
+
+sed -i "s|$prev_npm_hash|$new_npm_hash|g" "$PACKAGE_DIR/frontend.nix"
+
+# pdfjs version
+pdfjs_version="$(grep 'PDFJS_VERSION=' "$src/Dockerfile" | cut -d'=' -f2)"
+
+sed -i "s|pdfjsVersion = .*;|pdfjsVersion = \"$pdfjs_version\";|" "$PACKAGE_DIR/frontend.nix"
+
+# pdfjs hash
+sed -i "s|pdfjsHash = .*;|pdfjsHash = \"sha256-AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=\";|" "$PACKAGE_DIR/frontend.nix"
+
+set +e
+new_pdfjs_hash="$(
+  nix-build --no-out-link -A pdfding.frontend.pdfjs "$NIXPKGS_PATH" 2>&1 >/dev/null | grep "got:" | cut -d':' -f2 | sed 's| ||g'
+)"
+set -e
+
+sed -i "s|\"sha256-AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=\"|\"$new_pdfjs_hash\"|g" "$PACKAGE_DIR/frontend.nix"