Merge pull request #215110 from gador/pgadmin-use-pstgresqltesthook

pgadmin4-desktopmode: init at 6.19, pgadmin4: simplify tests

nixos/tests/all-tests.nix

@@ -520,7 +520,6 @@ in {
   peering-manager = handleTest ./web-apps/peering-manager.nix {};
   peertube = handleTestOn ["x86_64-linux"] ./web-apps/peertube.nix {};
   pgadmin4 = handleTest ./pgadmin4.nix {};
-  pgadmin4-standalone = handleTest ./pgadmin4-standalone.nix {};
   pgjwt = handleTest ./pgjwt.nix {};
   pgmanage = handleTest ./pgmanage.nix {};
   phosh = handleTest ./phosh.nix {};

nixos/tests/pgadmin4-standalone.nix

@@ -1,43 +0,0 @@
-import ./make-test-python.nix ({ pkgs, lib, ... }:
-  # This is separate from pgadmin4 since we don't want both running at once
-
-  {
-    name = "pgadmin4-standalone";
-    meta.maintainers = with lib.maintainers; [ mkg20001 ];
-
-    nodes.machine = { pkgs, ... }: {
-      environment.systemPackages = with pkgs; [
-        curl
-      ];
-
-      services.postgresql = {
-        enable = true;
-
-        authentication = ''
-          host    all             all             localhost               trust
-        '';
-
-        ensureUsers = [
-          {
-            name = "postgres";
-            ensurePermissions = {
-              "DATABASE \"postgres\"" = "ALL PRIVILEGES";
-            };
-          }
-        ];
-      };
-
-      services.pgadmin = {
-        enable = true;
-        initialEmail = "bruh@localhost.de";
-        initialPasswordFile = pkgs.writeText "pw" "bruh2012!";
-      };
-    };
-
-    testScript = ''
-      machine.wait_for_unit("postgresql")
-      machine.wait_for_unit("pgadmin")
-
-      machine.wait_until_succeeds("curl -s localhost:5050")
-    '';
-  })

nixos/tests/pgadmin4.nix

@@ -1,56 +1,18 @@
-import ./make-test-python.nix ({ pkgs, lib, buildDeps ? [ ], pythonEnv ? [ ], ... }:
+import ./make-test-python.nix ({ pkgs, lib, ... }:
 
-/*
-  This test suite replaces the typical pytestCheckHook function in python
-  packages. Pgadmin4 test suite needs a running and configured postgresql
-  server. This is why this test exists.
-
-  To not repeat all the python dependencies needed, this test is called directly
-  from the pgadmin4 derivation, which also passes the currently
-  used propagatedBuildInputs and any python overrides.
-
-  Unfortunately, there doesn't seem to be an easy way to otherwise include
-  the needed packages here.
-
-  Due the the needed parameters a direct call to "nixosTests.pgadmin4" fails
-  and needs to be called as "pgadmin4.tests"
-
-  */
-
-let
-  pgadmin4SrcDir = "/pgadmin";
-  pgadmin4Dir = "/var/lib/pgadmin";
-  pgadmin4LogDir = "/var/log/pgadmin";
-
-in
 {
   name = "pgadmin4";
-  meta.maintainers = with lib.maintainers; [ gador ];
+  meta.maintainers = with lib.maintainers; [ mkg20001 gador ];
 
   nodes.machine = { pkgs, ... }: {
-    imports = [ ./common/x11.nix ];
+
-    # needed because pgadmin 6.8 will fail, if those dependencies get updated
+    imports = [ ./common/user-account.nix ];
-    nixpkgs.overlays = [
-      (self: super: {
-        pythonPackages = pythonEnv;
-      })
-    ];
 
     environment.systemPackages = with pkgs; [
-      pgadmin4
+      curl
-      postgresql
+      pgadmin4-desktopmode
-      chromedriver
-      chromium
-      # include the same packages as in pgadmin minus speaklater3
-      (python3.withPackages
-        (ps: buildDeps ++
-          [
-            # test suite package requirements
-            pythonPackages.testscenarios
-            pythonPackages.selenium
-          ])
-      )
     ];
+
     services.postgresql = {
       enable = true;
       authentication = ''
@@ -65,75 +27,31 @@ in
         }
       ];
     };
+
+    services.pgadmin = {
+      port = 5051;
+      enable = true;
+      initialEmail = "bruh@localhost.de";
+      initialPasswordFile = pkgs.writeText "pw" "bruh2012!";
+    };
   };
 
   testScript = ''
+    with subtest("Check pgadmin module"):
       machine.wait_for_unit("postgresql")
+      machine.wait_for_unit("pgadmin")
+      machine.wait_until_succeeds("curl -s localhost:5051")
+      machine.wait_until_succeeds("curl -s localhost:5051/login | grep \"<title>pgAdmin 4</title>\" > /dev/null")
 
-    # pgadmin4 needs its data and log directories
+    # pgadmin4 module saves the configuration to /etc/pgadmin/config_system.py
-    machine.succeed(
+    # pgadmin4-desktopmode tries to read that as well. This normally fails with a PermissionError, as the config file
-        "mkdir -p ${pgadmin4Dir} \
+    # is owned by the user of the pgadmin module. With the check-system-config-dir.patch this will just throw a warning
-        && mkdir -p ${pgadmin4LogDir} \
+    # but will continue and not read the file.
-        && mkdir -p ${pgadmin4SrcDir}"
+    # If we run pgadmin4-desktopmode as root (something one really shouldn't do), it can read the config file and fail,
-    )
+    # because of the wrong config for desktopmode.
-
+    with subtest("Check pgadmin standalone desktop mode"):
-    machine.succeed(
+      machine.execute("sudo -u alice pgadmin4 >&2 &", timeout=60)
-         "tar xvzf ${pkgs.pgadmin4.src} -C ${pgadmin4SrcDir}"
+      machine.wait_until_succeeds("curl -s localhost:5050")
-    )
+      machine.wait_until_succeeds("curl -s localhost:5050/browser/ | grep \"<title>pgAdmin 4</title>\" > /dev/null")
-
-    machine.wait_for_file("${pgadmin4SrcDir}/pgadmin4-${pkgs.pgadmin4.version}/README.md")
-
-    # set paths and config for tests
-    # also ensure Server Mode is set to false, which will automatically exclude some unnecessary tests.
-    # see https://github.com/pgadmin-org/pgadmin4/blob/fd1c26408bbf154fa455a49ee5c12895933833a3/web/regression/runtests.py#L217-L226
-    machine.succeed(
-        "cd ${pgadmin4SrcDir}/pgadmin4-${pkgs.pgadmin4.version} \
-        && cp -v web/regression/test_config.json.in web/regression/test_config.json \
-        && sed -i 's|PostgreSQL 9.4|PostgreSQL|' web/regression/test_config.json \
-        && sed -i 's|/opt/PostgreSQL/9.4/bin/|${pkgs.postgresql}/bin|' web/regression/test_config.json \
-        && sed -i 's|\"headless_chrome\": false|\"headless_chrome\": true|' web/regression/test_config.json \
-        && sed -i 's|builtins.SERVER_MODE = None|builtins.SERVER_MODE = False|' web/regression/runtests.py"
-    )
-
-    # adapt chrome config to run within a sandbox without GUI
-    # see https://stackoverflow.com/questions/50642308/webdriverexception-unknown-error-devtoolsactiveport-file-doesnt-exist-while-t#50642913
-    # add chrome binary path. use spaces to satisfy python indention (tabs throw an error)
-    machine.succeed(
-         "cd ${pgadmin4SrcDir}/pgadmin4-${pkgs.pgadmin4.version} \
-         && sed -i '\|options.add_argument(\"--disable-infobars\")|a \ \ \ \ \ \ \ \ options.binary_location = \"${pkgs.chromium}/bin/chromium\"' web/regression/runtests.py \
-         && sed -i '\|options.add_argument(\"--no-sandbox\")|a \ \ \ \ \ \ \ \ options.add_argument(\"--headless\")' web/regression/runtests.py \
-         && sed -i '\|options.add_argument(\"--disable-infobars\")|a \ \ \ \ \ \ \ \ options.add_argument(\"--disable-dev-shm-usage\")' web/regression/runtests.py \
-         && sed -i 's|(chrome_options=options)|(executable_path=\"${pkgs.chromedriver}/bin/chromedriver\", chrome_options=options)|' web/regression/runtests.py \
-         && sed -i 's|driver_local.maximize_window()||' web/regression/runtests.py"
-    )
-
-    # don't bother to test kerberos authentication
-    excluded_tests = [ "browser.tests.test_kerberos_with_mocking",
-                       ]
-
-    with subtest("run browser test"):
-        machine.succeed(
-             'cd ${pgadmin4SrcDir}/pgadmin4-${pkgs.pgadmin4.version}/web \
-             && python regression/runtests.py \
-             --pkg browser \
-             --exclude ' + ','.join(excluded_tests)
-        )
-
-    with subtest("run resql test"):
-      machine.succeed(
-           'cd ${pgadmin4SrcDir}/pgadmin4-${pkgs.pgadmin4.version}/web \
-           && python regression/runtests.py --pkg resql'
-      )
-
-    # fontconfig is necessary for chromium to run
-    # https://github.com/NixOS/nixpkgs/issues/136207
-    # also, the feature_tests require Server Mode = True
-    with subtest("run feature test"):
-       machine.succeed(
-           'cd ${pgadmin4SrcDir}/pgadmin4-${pkgs.pgadmin4.version}/web \
-            && export FONTCONFIG_FILE=${pkgs.makeFontsConf { fontDirectories = [];}} \
-            && sed -i \'s|builtins.SERVER_MODE = False|builtins.SERVER_MODE = True|\' regression/runtests.py \
-            && python regression/runtests.py --pkg feature_tests'
-       )
   '';
 })

pkgs/tools/admin/pgadmin/check-system-config-dir.patch

@@ -0,0 +1,17 @@
+diff --git a/web/config.py b/web/config.py
+index 4774043..5b73fd3 100644
+--- a/web/config.py
++++ b/web/config.py
+@@ -884,6 +884,12 @@ if os.path.exists(system_config_dir + '/config_system.py'):
+         user_config_settings.update(config_system_settings)
+     except ImportError:
+         pass
++    except PermissionError:
++        print(f"Permission denied to open {str(system_config_dir + '/config_system.py')}. \n \
++              If you are running pgadmin4-desktopmode please make sure you disable  \n \
++              the pgadmin NixOS module first. If you rely on settings in \n \
++              {str(system_config_dir + '/config_system.py')}, please check the correct permissions.")
++        pass
+
+ # Update settings for 'LOG_FILE', 'SQLITE_PATH', 'SESSION_DB_PATH',
+ # 'AZURE_CREDENTIAL_CACHE_DIR', 'KERBEROS_CCACHE_DIR', 'STORAGE_DIR'

pkgs/tools/admin/pgadmin/default.nix

@@ -7,6 +7,9 @@
 , nixosTests
 , pkgs
 , fetchPypi
+, postgresqlTestHook
+, postgresql
+, server-mode ? true
 }:
 
 let
@@ -26,54 +29,6 @@ let
     yarnNix = ./yarn.nix;
   };
 
-  # move buildDeps here to easily pass to test suite
-  buildDeps = with pythonPackages; [
-    flask
-    flask-gravatar
-    flask-login
-    flask_mail
-    flask_migrate
-    flask-sqlalchemy
-    flask-wtf
-    flask-compress
-    passlib
-    pytz
-    simplejson
-    sqlparse
-    wtforms
-    flask-paranoid
-    psutil
-    psycopg2
-    python-dateutil
-    sqlalchemy
-    itsdangerous
-    flask-security-too
-    bcrypt
-    cryptography
-    sshtunnel
-    ldap3
-    flask-babelex
-    flask-babel
-    gssapi
-    flask-socketio
-    eventlet
-    httpagentparser
-    user-agents
-    wheel
-    authlib
-    qrcode
-    pillow
-    pyotp
-    botocore
-    boto3
-    azure-mgmt-subscription
-    azure-mgmt-rdbms
-    azure-mgmt-resource
-    azure-identity
-    sphinxcontrib-youtube
-    dnspython
-    greenlet
-  ];
 
   # keep the scope, as it is used throughout the derivation and tests
   # this also makes potential future overrides easier
@@ -134,6 +89,8 @@ pythonPackages.buildPythonApplication rec {
   patches = [
     # Expose setup.py for later use
     ./expose-setup.py.patch
+    # check for permission of /etc/pgadmin/config_system and don't fail
+    ./check-system-config-dir.patch
   ];
 
   postPatch = ''
@@ -149,10 +106,12 @@ pythonPackages.buildPythonApplication rec {
 
     # relax dependencies
     sed 's|==|>=|g' -i requirements.txt
-    # don't use Server Mode (can be overridden later)
     substituteInPlace pkg/pip/setup_pip.py \
-      --replace "req = req.replace('psycopg2', 'psycopg2-binary')" "req = req" \
+      --replace "req = req.replace('psycopg2', 'psycopg2-binary')" "req = req"
-      --replace "builtins.SERVER_MODE = None" "builtins.SERVER_MODE = False"
+    ${lib.optionalString (!server-mode) ''
+    substituteInPlace web/config.py \
+      --replace "SERVER_MODE = True" "SERVER_MODE = False"
+    ''}
   '';
 
   preBuild = ''
@@ -200,23 +159,110 @@ pythonPackages.buildPythonApplication rec {
     pythonPackages.wheel
   ];
 
-  # tests need an own data, log directory
+  propagatedBuildInputs = with pythonPackages; [
-  # and a working and correctly setup postgres database
+    flask
-  # checks will be run through nixos/tests
+    flask-gravatar
-  doCheck = false;
+    flask-login
-
+    flask_mail
-  # speaklater3 is separate because when passing buildDeps
+    flask_migrate
-  # to the test, it fails there due to a collision with speaklater
+    flask-sqlalchemy
-  propagatedBuildInputs = buildDeps ++ [ pythonPackages.speaklater3 ];
+    flask-wtf
+    flask-compress
+    passlib
+    pytz
+    simplejson
+    sqlparse
+    wtforms
+    flask-paranoid
+    psutil
+    psycopg2
+    python-dateutil
+    sqlalchemy
+    itsdangerous
+    flask-security-too
+    bcrypt
+    cryptography
+    sshtunnel
+    ldap3
+    flask-babelex
+    flask-babel
+    gssapi
+    flask-socketio
+    eventlet
+    httpagentparser
+    user-agents
+    wheel
+    authlib
+    qrcode
+    pillow
+    pyotp
+    botocore
+    boto3
+    azure-mgmt-subscription
+    azure-mgmt-rdbms
+    azure-mgmt-resource
+    azure-identity
+    sphinxcontrib-youtube
+    dnspython
+    greenlet
+    speaklater3
+  ];
 
   passthru.tests = {
-    standalone = nixosTests.pgadmin4-standalone;
+    inherit (nixosTests) pgadmin4;
-    # regression and function tests of the package itself
-    package = import ../../../../nixos/tests/pgadmin4.nix { inherit pkgs buildDeps; pythonEnv = pythonPackages; };
   };
 
+  nativeCheckInputs = [
+    postgresqlTestHook
+    postgresql
+    pythonPackages.testscenarios
+    pythonPackages.selenium
+  ];
+
+  checkPhase = ''
+    runHook preCheck
+
+    ## Setup ##
+
+    # pgadmin needs a home directory to save the configuration
+    export HOME=$TMPDIR
+    cd pgadmin4
+
+    # set configuration for postgresql test
+    # also ensure Server Mode is set to false. If not, the tests will fail, since pgadmin expects read/write permissions
+    # in /var/lib/pgadmin and /var/log/pgadmin
+    # see https://github.com/pgadmin-org/pgadmin4/blob/fd1c26408bbf154fa455a49ee5c12895933833a3/web/regression/runtests.py#L217-L226
+    cp -v regression/test_config.json.in regression/test_config.json
+    substituteInPlace regression/test_config.json --replace "localhost" "$PGHOST"
+    substituteInPlace regression/runtests.py --replace "builtins.SERVER_MODE = None" "builtins.SERVER_MODE = False"
+
+    ## Browser test ##
+
+    # don't bother to test kerberos authentication
+    python regression/runtests.py --pkg browser --exclude browser.tests.test_kerberos_with_mocking
+
+    ## Reverse engineered SQL test ##
+
+    python regression/runtests.py --pkg resql
+
+    runHook postCheck
+  '';
+
   meta = with lib; {
-    description = "Administration and development platform for PostgreSQL";
+    description = "Administration and development platform for PostgreSQL${optionalString (!server-mode) ". Desktop Mode"}";
+    longDescription = ''
+      pgAdmin 4 is designed to meet the needs of both novice and experienced Postgres users alike,
+      providing a powerful graphical interface that simplifies the creation, maintenance and use of database objects.
+      ${if server-mode then ''
+      This version is build with SERVER_MODE set to True (the default). It will require access to `/var/lib/pgadmin`
+      and `/var/log/pgadmin`. This is the default version for the NixOS module `services.pgadmin`.
+      This should NOT be used in combination with the `pgadmin4-desktopmode` package as they will interfere.
+      '' else ''
+      This version is build with SERVER_MODE set to False. It will require access to `~/.pgadmin/`. This version is suitable
+      for single-user deployment or where access to `/var/lib/pgadmin` cannot be granted or the NixOS module cannot be used.
+      This should NOT be used in combination with the NixOS module `pgadmin` as they will interfere.
+      ''}
+    '';
     homepage = "https://www.pgadmin.org/";
     license = licenses.mit;
     changelog = "https://www.pgadmin.org/docs/pgadmin4/latest/release_notes_${lib.versions.major version}_${lib.versions.minor version}.html";

pkgs/top-level/all-packages.nix

@@ -38043,6 +38043,8 @@ with pkgs;
 
   pgadmin4 = callPackage ../tools/admin/pgadmin { };
 
+  pgadmin4-desktopmode = callPackage ../tools/admin/pgadmin { server-mode = false; };
+
   pgmodeler = qt6Packages.callPackage ../applications/misc/pgmodeler { };
 
   physlock = callPackage ../misc/screensavers/physlock { };