Merge master into staging-next

2022-04-18 12:01:22 +00:00
parent 206c8a30ba e00c10bf0d
commit 35e69ca743
22 changed files with 335 additions and 23 deletions
@@ -0,0 +1,10 @@
+<chapter xmlns="http://docbook.org/ns/docbook"
+         xmlns:xlink="http://www.w3.org/1999/xlink"
+         xmlns:xi="http://www.w3.org/2001/XInclude"
+         xml:id="chap-hooks">
+ <title>Hooks reference</title>
+ <para>
+  Nixpkgs has several hook packages that augment the stdenv phases.
+ </para>
+ <xi:include href="./postgresql-test-hook.section.xml" />
+</chapter>
@@ -0,0 +1,59 @@
+
+# `postgresqlTestHook` {#sec-postgresqlTestHook}
+
+This hook starts a PostgreSQL server during the `checkPhase`. Example:
+
+```nix
+{ stdenv, postgresql, postgresqlTestHook }:
+stdenv.mkDerivation {
+
+  # ...
+
+  checkInputs = [
+    postgresql
+    postgresqlTestHook
+  ];
+}
+```
+
+If you use a custom `checkPhase`, remember to add the `runHook` calls:
+```nix
+  checkPhase ''
+    runHook preCheck
+
+    # ... your tests
+
+    runHook postCheck
+  ''
+```
+
+## Variables {#sec-postgresqlTestHook-variables}
+
+The hook logic will read a number of variables and set them to a default value if unset or empty.
+
+Exported variables:
+
+ - `PGDATA`: location of server files.
+ - `PGHOST`: location of UNIX domain socket directory; the default `host` in a connection string.
+ - `PGUSER`: user to create / log in with, default: `test_user`.
+ - `PGDATABASE`: database name, default: `test_db`.
+
+Bash-only variables:
+
+ - `postgresqlTestUserOptions`: SQL options to use when creating the `$PGUSER` role, default: `LOGIN`.
+ - `postgresqlTestSetupSQL`: SQL commands to run as database administrator after startup, default: statements that create `$PGUSER` and `$PGDATABASE`.
+ - `postgresqlTestSetupCommands`: bash commands to run after database start, defaults to running `$postgresqlTestSetupSQL` as database administrator.
+ - `postgresqlEnableTCP`: set to `1` to enable TCP listening. Flaky; not recommended.
+ - `postgresqlStartCommands`: defaults to `pg_ctl start`.
+
+## TCP and the Nix sandbox {#sec-postgresqlTestHook-tcp}
+
+`postgresqlEnableTCP` relies on network sandboxing, which is not available on macOS and some custom Nix installations, resulting in flaky tests.
+For this reason, it is disabled by default.
+
+The preferred solution is to make the test suite use a UNIX domain socket connection. This is the default behavior when no `host` connection parameter is provided.
+Some test suites hardcode a value for `host` though, so a patch may be required. If you can upstream the patch, you can make `host` default to the `PGHOST` environment variable when set. Otherwise, you can patch it locally to omit the `host` connection string parameter altogether.
+
+::: {.note}
+The error `libpq: failed (could not receive data from server: Connection refused` is generally an indication that the test suite is trying to connect through TCP.
+:::
@@ -27,6 +27,7 @@
  <xi:include href="builders/trivial-builders.chapter.xml" />
  <xi:include href="builders/special.xml" />
  <xi:include href="builders/images.xml" />
+  <xi:include href="hooks/index.xml" />
  <xi:include href="languages-frameworks/index.xml" />
  <xi:include href="builders/packages/index.xml" />
 </part>
@@ -73,6 +73,13 @@
          Systemd has been upgraded to the version 250.
        </para>
      </listitem>
+      <listitem>
+        <para>
+          The new
+          <link xlink:href="https://nixos.org/manual/nixpkgs/stable/#sec-postgresqlTestHook"><literal>postgresqlTestHook</literal></link>
+          runs a PostgreSQL server for the duration of package checks.
+        </para>
+      </listitem>
      <listitem>
        <para>
          <link xlink:href="https://kops.sigs.k8s.io"><literal>kops</literal></link>
@@ -1238,6 +1245,14 @@
          <literal>systemd.nspawn.&lt;name&gt;.execConfig.PrivateUsers = false</literal>
        </para>
      </listitem>
+      <listitem>
+        <para>
+          <literal>systemd-shutdown</literal> is now properly linked on
+          shutdown to unmount all filesystems and device mapper devices
+          cleanly. This can be disabled using
+          <literal>boot.systemd.shutdown.enable</literal>.
+        </para>
+      </listitem>
      <listitem>
        <para>
          The Tor SOCKS proxy is now actually disabled if
@@ -27,6 +27,8 @@ In addition to numerous new and upgraded packages, this release has the followin

 - Systemd has been upgraded to the version 250.

+- The new [`postgresqlTestHook`](https://nixos.org/manual/nixpkgs/stable/#sec-postgresqlTestHook) runs a PostgreSQL server for the duration of package checks.
+
 - [`kops`](https://kops.sigs.k8s.io) defaults to 1.22.4, which will enable [Instance Metadata Service Version 2](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/configuring-instance-metadata-service.html) and require tokens on new clusters with Kubernetes 1.22. This will increase security by default, but may break some types of workloads. See the [release notes](https://kops.sigs.k8s.io/releases/1.22-notes/) for details.

 - Module authors can use `mkRenamedOptionModuleWith` to automate the deprecation cycle without annoying out-of-tree module authors and their users.
@@ -492,6 +494,8 @@ In addition to numerous new and upgraded packages, this release has the followin

 - `systemd-nspawn@.service` settings have been reverted to the default systemd behaviour. User namespaces are now activated by default. If you want to keep running nspawn containers without user namespaces you need to set `systemd.nspawn.<name>.execConfig.PrivateUsers = false`

+- `systemd-shutdown` is now properly linked on shutdown to unmount all filesystems and device mapper devices cleanly. This can be disabled using `boot.systemd.shutdown.enable`.
+
 - The Tor SOCKS proxy is now actually disabled if `services.tor.client.enable` is set to `false` (the default). If you are using this functionality but didn't change the setting or set it to `false`, you now need to set it to `true`.

 - The terraform 0.12 compatibility has been removed and the `terraform.withPlugins` and `terraform-providers.mkProvider` implementations simplified. Providers now need to be stored under
@@ -1183,6 +1183,7 @@
  ./system/boot/systemd/journald.nix
  ./system/boot/systemd/logind.nix
  ./system/boot/systemd/nspawn.nix
+  ./system/boot/systemd/shutdown.nix
  ./system/boot/systemd/tmpfiles.nix
  ./system/boot/systemd/user.nix
  ./system/boot/systemd/initrd.nix
@@ -0,0 +1,32 @@
+{ config, lib, ... }: let
+
+  cfg = config.boot.systemd.shutdown;
+
+in {
+  options.boot.systemd.shutdown = {
+    enable = lib.mkEnableOption "pivoting back to an initramfs for shutdown" // { default = true; };
+  };
+
+  config = lib.mkIf cfg.enable {
+    systemd.services.generate-shutdown-ramfs = {
+      description = "Generate shutdown ramfs";
+      before = [ "shutdown.target" ];
+      unitConfig = {
+        DefaultDependencies = false;
+        ConditionFileIsExecutable = [
+          "!/run/initramfs/shutdown"
+          "/run/current-system/systemd/lib/systemd/systemd-shutdown"
+        ];
+      };
+
+      serviceConfig.Type = "oneshot";
+      script = ''
+        mkdir -p /run/initramfs
+        if ! mountpoint -q /run/initramfs; then
+          mount -t tmpfs tmpfs /run/initramfs
+        fi
+        cp /run/current-system/systemd/lib/systemd/systemd-shutdown /run/initramfs/shutdown
+      '';
+    };
+  };
+}
@@ -524,6 +524,7 @@ in
  systemd-confinement = handleTest ./systemd-confinement.nix {};
  systemd-cryptenroll = handleTest ./systemd-cryptenroll.nix {};
  systemd-escaping = handleTest ./systemd-escaping.nix {};
+  systemd-initrd-shutdown = handleTest ./systemd-shutdown.nix { systemdStage1 = true; };
  systemd-initrd-simple = handleTest ./systemd-initrd-simple.nix {};
  systemd-initrd-swraid = handleTest ./systemd-initrd-swraid.nix {};
  systemd-journal = handleTest ./systemd-journal.nix {};
@@ -534,6 +535,7 @@ in
  systemd-networkd-ipv6-prefix-delegation = handleTest ./systemd-networkd-ipv6-prefix-delegation.nix {};
  systemd-networkd-vrf = handleTest ./systemd-networkd-vrf.nix {};
  systemd-nspawn = handleTest ./systemd-nspawn.nix {};
+  systemd-shutdown = handleTest ./systemd-shutdown.nix {};
  systemd-timesyncd = handleTest ./systemd-timesyncd.nix {};
  systemd-misc = handleTest ./systemd-misc.nix {};
  taskserver = handleTest ./taskserver.nix {};
@@ -0,0 +1,21 @@
+import ./make-test-python.nix ({ pkgs, systemdStage1 ? false, ...} : {
+  name = "systemd-shutdown";
+  meta = with pkgs.lib.maintainers; {
+    maintainers = [ das_j ];
+  };
+
+  nodes.machine = {
+    imports = [ ../modules/profiles/minimal.nix ];
+    boot.initrd.systemd.enable = systemdStage1;
+  };
+
+  testScript = ''
+    machine.wait_for_unit("multi-user.target")
+    # .shutdown() would wait for the machine to power off
+    machine.succeed("systemctl poweroff")
+    # Message printed by systemd-shutdown
+    machine.wait_for_console_text("All filesystems, swaps, loop devices, MD devices and DM devices detached.")
+    # Don't try to sync filesystems
+    machine.booted = False
+  '';
+})
@@ -5,14 +5,14 @@

 mkDerivation rec {
  pname = "qpwgraph";
-  version = "0.2.2";
+  version = "0.2.5";

  src = fetchFromGitLab {
    domain = "gitlab.freedesktop.org";
    owner = "rncbc";
    repo = "qpwgraph";
    rev = "v${version}";
-    sha256 = "sha256-BBvF1L3IqkYqSghHxcbwOBizdu6GtxaWof3Q/bc+aTY=";
+    sha256 = "sha256-OYIBlTO1vXmmY4/ZacvsEQ5EnOfetBvnG2v5xL44czY=";
  };

  nativeBuildInputs = [ cmake pkg-config ];
@@ -29,6 +29,6 @@ mkDerivation rec {
    homepage = "https://gitlab.freedesktop.org/rncbc/qpwgraph";
    license = licenses.gpl2Plus;
    platforms = platforms.linux;
-    maintainers = with maintainers; [ kanashimia ];
+    maintainers = with maintainers; [ kanashimia exi ];
  };
 }
@@ -20,8 +20,8 @@ let
 in
 python3Packages.buildPythonApplication rec {
  pname = "bottles";
-  version = "2022.4.14-trento";
-  sha256 = "0kjc1w8x4d6g2lx8x8isa2vnwacyjlh9lldf14wn4b118hsw85zs";
+  version = "2022.4.14-trento-1";
+  sha256 = "16cb01fhxa64f8fadwpr0mawfmchig6xlbx20mz4q9yh5fnagywj";
  # Note: Update via pkgs/applications/misc/bottles/update.py
  # mostly copypasted from pkgs/applications/networking/instant-messengers/telegram/tdesktop/update.py

@@ -3,20 +3,20 @@
 }:
 let
  pname = "josm";
-  version = "18387";
+  version = "18427";
  srcs = {
    jar = fetchurl {
      url = "https://josm.openstreetmap.de/download/josm-snapshot-${version}.jar";
-      sha256 = "sha256-zkEWZjjSy0ajG5X1/dIgLPZ7zr0BiaJJcHaN8sv/3yc=";
+      sha256 = "sha256-SsoeKfpWi41sd77LfaQW0OcHnyf8iLolKWF74dhalpY=";
    };
    macosx = fetchurl {
      url = "https://josm.openstreetmap.de/download/macosx/josm-macos-${version}-java17.zip";
-      sha256 = "sha256-xOVnnQ+eUkboT8Tq5F4QJEou1wAaHwiEdyiEKDR/fUk=";
+      sha256 = "sha256-cz3OT03StvJy9XYBiPxx+nz36O0cg+XrL/uc52/G/1c=";
    };
    pkg = fetchsvn {
      url = "https://josm.openstreetmap.de/svn/trunk/native/linux/tested";
      rev = version;
-      sha256 = "sha256-GbIWZTJmmUT8r9L63/6mcnRt7dvavqGAVbozxlbF89g=";
+      sha256 = "sha256-AyfUFE0y8d8FtgfhxSscQGE1IVNwYYNymH6jTnrUBTs=";
    };
  };
 in
@@ -12,7 +12,7 @@ in
  pdfstudio = callPackage ./common.nix rec {
    pname = program;
    year = "2021";
-    version = "${year}.1.2";
+    version = "${year}.1.3";
    desktopName = "PDF Studio";
    longDescription = ''
      PDF Studio is an easy to use, full-featured PDF editing software. This is the standard/pro edition, which requires a license. For the free PDF Studio Viewer see the package pdfstudioviewer.
@@ -22,21 +22,21 @@ in
    ];
    src = fetchurl {
      url = makeurl { inherit pname year version; };
-      sha256 = "1188ll2qz58rr2slavqxisbz4q3fdzidpasb1p33926z0ym3rk45";
+      sha256 = "sha256-2az8/slWeLY7l7dCwyTaT18KFfvsO71vJFDZEvbDHGM=";
    };
  };

  pdfstudioviewer = callPackage ./common.nix rec {
    pname = program;
    year = "2021";
-    version = "${year}.1.2";
+    version = "${year}.1.3";
    desktopName = "PDF Studio Viewer";
    longDescription = ''
      PDF Studio Viewer is an easy to use, full-featured PDF editing software. This is the free edition. For the standard/pro edition, see the package pdfstudio.
    '';
    src = fetchurl {
      url = makeurl { inherit pname year version; };
-      sha256 = "128k3fm8m8zdykx4s30g5m2zl7cgmvs4qinf1w525zh84v56agz6";
+      sha256 = "sha256-kd+rQruBL0fudmc30agRO/XV97l/6unqU0GK25yhOzI=";
    };
  };
 }.${program}
@@ -1,4 +1,4 @@
-{ lib, stdenv, fetchFromGitHub, openssl, libsamplerate, alsa-lib, AppKit }:
+{ lib, stdenv, fetchFromGitHub, openssl, libsamplerate, alsa-lib, AppKit, fetchpatch }:

 stdenv.mkDerivation rec {
  pname = "pjsip";
@@ -13,6 +13,11 @@ stdenv.mkDerivation rec {

  patches = [
    ./fix-aarch64.patch
+    (fetchpatch {
+      name = "CVE-2022-24764.patch";
+      url = "https://github.com/pjsip/pjproject/commit/560a1346f87aabe126509bb24930106dea292b00.patch";
+      sha256 = "1fy78v3clm0gby7qcq3ny6f7d7f4qnn01lkqq67bf2s85k2phisg";
+    })
  ];

  buildInputs = [ openssl libsamplerate ]
@@ -7,11 +7,11 @@

 buildPythonApplication rec {
  pname = "ffmpeg-normalize";
-  version = "1.22.8";
+  version = "1.22.9";

  src = fetchPypi {
    inherit pname version;
-    sha256 = "sha256-vxiq6q8fPh8ZLKnxYdIN591UQW73FWsoke1PvKTkko8=";
+    sha256 = "sha256-RBrCIDinPXbXKqrrhqVf3rV4rfi+2PttIaYxUKOk7hs=";
  };

  propagatedBuildInputs = [ ffmpeg ffmpeg-progress-yield ];
@@ -0,0 +1,9 @@
+{ callPackage, makeSetupHook }:
+
+(makeSetupHook {
+  name = "postgresql-test-hook";
+} ./postgresql-test-hook.sh).overrideAttrs (o: {
+  passthru.tests = {
+    simple = callPackage ./test.nix { };
+  };
+})
@@ -0,0 +1,79 @@
+preCheckHooks+=('postgresqlStart')
+postCheckHooks+=('postgresqlStop')
+
+
+postgresqlStart() {
+
+  # Add default environment variable values
+  #
+  # Client variables:
+  #  - https://www.postgresql.org/docs/current/libpq-envars.html
+  #
+  # Server variables:
+  #  - only PGDATA: https://www.postgresql.org/docs/current/creating-cluster.html
+
+  if [[ "${PGDATA:-}" == "" ]]; then
+    PGDATA="$NIX_BUILD_TOP/postgresql"
+  fi
+  export PGDATA
+
+  if [[ "${PGHOST:-}" == "" ]]; then
+    mkdir -p "$NIX_BUILD_TOP/run/postgresql"
+    PGHOST="$NIX_BUILD_TOP/run/postgresql"
+  fi
+  export PGHOST
+
+  if [[ "${PGUSER:-}" == "" ]]; then
+    PGUSER="test_user"
+  fi
+  export PGUSER
+
+  if [[ "${PGDATABASE:-}" == "" ]]; then
+    PGDATABASE="test_db"
+  fi
+  export PGDATABASE
+
+  if [[ "${postgresqlTestUserOptions:-}" == "" ]]; then
+    postgresqlTestUserOptions="LOGIN"
+  fi
+
+  if [[ "${postgresqlTestSetupSQL:-}" == "" ]]; then
+    postgresqlTestSetupSQL="$(cat <<EOF
+      CREATE ROLE "$PGUSER" $postgresqlTestUserOptions;
+      CREATE DATABASE "$PGDATABASE" OWNER '$PGUSER';
+EOF
+    )"
+  fi
+
+  if [[ "${postgresqlTestSetupCommands:-}" == "" ]]; then
+    postgresqlTestSetupCommands='echo "$postgresqlTestSetupSQL" | PGUSER=postgres psql postgres'
+  fi
+
+  if ! type initdb >/dev/null; then
+    echo >&2 'initdb not found. Did you add postgresql to the checkInputs?'
+    false
+  fi
+  header 'initializing postgresql'
+  initdb -U postgres
+
+  # Move the socket
+  echo "unix_socket_directories = '$NIX_BUILD_TOP/run/postgresql'" >>"$PGDATA/postgresql.conf"
+
+  # TCP ports can be a problem in some sandboxes,
+  # so we disable tcp listening by default
+  if ! [[ "${postgresqlEnableTCP:-}" = 1 ]]; then
+    echo "listen_addresses = ''" >>"$PGDATA/postgresql.conf"
+  fi
+
+  header 'starting postgresql'
+  eval "${postgresqlStartCommands:-pg_ctl start}"
+
+  header 'setting up postgresql'
+  eval "$postgresqlTestSetupCommands"
+
+}
+
+postgresqlStop() {
+  header 'stopping postgresql'
+  pg_ctl stop
+}
@@ -0,0 +1,27 @@
+{ postgresql, postgresqlTestHook, stdenv }:
+
+stdenv.mkDerivation {
+  name = "postgresql-test-hook-test";
+  buildInputs = [ postgresqlTestHook ];
+  checkInputs = [ postgresql ];
+  dontUnpack = true;
+  doCheck = true;
+  passAsFile = ["sql"];
+  sql = ''
+    CREATE TABLE hello (
+      message text
+    );
+    INSERT INTO hello VALUES ('it '||'worked');
+    SELECT * FROM hello;
+  '';
+  checkPhase = ''
+    runHook preCheck
+    psql <$sqlPath | grep 'it worked'
+    TEST_RAN=1
+    runHook postCheck
+  '';
+  installPhase = ''
+    [[ $TEST_RAN == 1 ]]
+    touch $out
+  '';
+}
@@ -1190,8 +1190,29 @@ self: super: {
  # Fix build with attr-2.4.48 (see #53716)
  xattr = appendPatch ./patches/xattr-fix-build.patch super.xattr;

-  # Some tests depend on a postgresql instance
-  esqueleto = dontCheck super.esqueleto;
+  esqueleto =
+    overrideCabal
+      (drv: {
+        postPatch = drv.postPatch or "" + ''
+          # patch out TCP usage: https://nixos.org/manual/nixpkgs/stable/#sec-postgresqlTestHook-tcp
+          sed -i test/PostgreSQL/Test.hs \
+            -e s^host=localhost^^
+        '';
+        # Match the test suite defaults (or hardcoded values?)
+        preCheck = drv.preCheck or "" + ''
+          PGUSER=esqutest
+          PGDATABASE=esqutest
+        '';
+        testFlags = drv.testFlags or [] ++ [
+          # We don't have a MySQL test hook yet
+          "--skip=/Esqueleto/MySQL"
+        ];
+        testToolDepends = drv.testToolDepends or [] ++ [
+          pkgs.postgresql
+          pkgs.postgresqlTestHook
+        ];
+      })
+      super.esqueleto;

  # Requires API keys to run tests
  algolia = dontCheck super.algolia;
@@ -1310,7 +1331,25 @@ self: super: {

  # Test suite requires database
  persistent-mysql = dontCheck super.persistent-mysql;
-  persistent-postgresql = dontCheck super.persistent-postgresql;
+  persistent-postgresql =
+    overrideCabal
+      (drv: {
+        postPatch = drv.postPath or "" + ''
+          # patch out TCP usage: https://nixos.org/manual/nixpkgs/stable/#sec-postgresqlTestHook-tcp
+          # NOTE: upstream host variable takes only two values...
+          sed -i test/PgInit.hs \
+            -e s^'host=" <> host <> "'^^
+        '';
+        preCheck = drv.preCheck or "" + ''
+          PGDATABASE=test
+          PGUSER=test
+        '';
+        testToolDepends = drv.testToolDepends or [] ++ [
+          pkgs.postgresql
+          pkgs.postgresqlTestHook
+        ];
+      })
+      super.persistent-postgresql;

  # Fix EdisonAPI and EdisonCore for GHC 8.8:
  # https://github.com/robdockins/edison/pull/16
@@ -1515,8 +1554,13 @@ self: super: {
  };
  pg-client = overrideCabal (drv: {
    librarySystemDepends = with pkgs; [ postgresql krb5.dev openssl.dev ];
-    # wants a running DB to check against
-    doCheck = false;
+    testToolDepends = drv.testToolDepends or [] ++ [
+      pkgs.postgresql pkgs.postgresqlTestHook
+    ];
+    preCheck = drv.preCheck or "" + ''
+      # empty string means use default connection
+      export DATABASE_URL=""
+    '';
  }) (super.pg-client.override {
    resource-pool = self.hasura-resource-pool;
    ekg-core = self.hasura-ekg-core;
@@ -16,11 +16,11 @@

 stdenv.mkDerivation rec {
  pname = "cypress";
-  version = "9.5.3";
+  version = "9.5.4";

  src = fetchzip {
    url = "https://cdn.cypress.io/desktop/${version}/linux-x64/cypress.zip";
-    sha256 = "qK1TvAGiTVmEQ4MpqICDSP9swnpAcE6iGkGJOrPQ1bI=";
+    sha256 = "F4BSIA3ImXwmmki8/FK0t08Gf5S8KMpXNNBIPPJQNsM=";
  };

  # don't remove runtime deps
@@ -165,6 +165,7 @@ in runBuildTests {

      [attrs]
      foo = "foo"
+
      [level1.level2.level3]
      level4 = "deep"
    '';
@@ -22181,6 +22181,8 @@ with pkgs;

  postgresql_jdbc = callPackage ../development/java-modules/postgresql_jdbc { };

+  postgresqlTestHook = callPackage ../build-support/setup-hooks/postgresql-test-hook { };
+
  prom2json = callPackage ../servers/monitoring/prometheus/prom2json.nix { };
  prometheus = callPackage ../servers/monitoring/prometheus { };
  prometheus-alertmanager = callPackage ../servers/monitoring/prometheus/alertmanager.nix { };