Merge pull request #334286 from nh2/ceph-18.2.4-staging-next-fix

ceph: Fix build by fully vendoring old cryptography version nix files.

pkgs/tools/filesystems/ceph/default.nix

@@ -142,7 +142,7 @@ let
      homepage = "https://ceph.io/en/";
      inherit description;
      license = with lib.licenses; [ lgpl21 gpl2Only bsd3 mit publicDomain ];
-     maintainers = with lib.maintainers; [ adev ak johanot krav ];
+     maintainers = with lib.maintainers; [ adev ak johanot krav nh2 ];
      platforms = [ "x86_64-linux" "aarch64-linux" ];
    };
 
@@ -172,12 +172,18 @@ let
   python = python311.override {
     self = python;
     packageOverrides = self: super: let
-      cryptographyOverrideVersion = "40.0.1";
       bcryptOverrideVersion = "4.0.1";
     in {
-      # Ceph does not support `bcrypt` > 4.0 yet:
+      # Ceph does not support the following yet:
+      # * `bcrypt` > 4.0
+      # * `cryptography` > 40
+      # See:
+      # * https://github.com/NixOS/nixpkgs/pull/281858#issuecomment-1899358602
       # * Upstream issue: https://tracker.ceph.com/issues/63529
       #   > Python Sub-Interpreter Model Used by ceph-mgr Incompatible With Python Modules Based on PyO3
+      # * Moved to issue: https://tracker.ceph.com/issues/64213
+      #   > MGR modules incompatible with later PyO3 versions - PyO3 modules may only be initialized once per interpreter process
+
       bcrypt = super.bcrypt.overridePythonAttrs (old: rec {
         pname = "bcrypt";
         version = bcryptOverrideVersion;
@@ -193,44 +199,12 @@ let
           hash = "sha256-lDWX69YENZFMu7pyBmavUZaalGvFqbHSHfkwkzmDQaY=";
         };
       });
-      # Ceph does not support `cryptography` > 40 yet:
-      # * https://github.com/NixOS/nixpkgs/pull/281858#issuecomment-1899358602
-      # * Upstream issue: https://tracker.ceph.com/issues/63529
-      #   > Python Sub-Interpreter Model Used by ceph-mgr Incompatible With Python Modules Based on PyO3
-      #
+
       # We pin the older `cryptography` 40 here;
       # this also forces us to pin an older `pyopenssl` because the current one
       # is not compatible with older `cryptography`, see:
       #     https://github.com/pyca/pyopenssl/blob/d9752e44127ba36041b045417af8a0bf16ec4f1e/CHANGELOG.rst#2320-2023-05-30
-      cryptography = super.cryptography.overridePythonAttrs (old: rec {
-        version = cryptographyOverrideVersion;
-
-        src = fetchPypi {
-          inherit (old) pname;
-          version = cryptographyOverrideVersion;
-          hash = "sha256-KAPy+LHpX2FEGZJsfm9V2CivxhTKXtYVQ4d65mjMNHI=";
-        };
-
-        cargoDeps = rustPlatform.fetchCargoTarball {
-          inherit src;
-          sourceRoot = let cargoRoot = "src/rust"; in "${old.pname}-${cryptographyOverrideVersion}/${cargoRoot}";
-          name = "${old.pname}-${cryptographyOverrideVersion}";
-          hash = "sha256-gFfDTc2QWBWHBCycVH1dYlCsWQMVcRZfOBIau+njtDU=";
-        };
-
-        # Not using the normal `(old.patches or []) ++` pattern here to use
-        # the overridden package's patches, because current nixpkgs's `cryptography`
-        # has patches that do not apply on this old version.
-        patches = [
-          # Fix https://nvd.nist.gov/vuln/detail/CVE-2023-49083 which has no upstream backport.
-          # See https://github.com/pyca/cryptography/commit/f09c261ca10a31fe41b1262306db7f8f1da0e48a#diff-f5134bf8f3cf0a5cc8601df55e50697acc866c603a38caff98802bd8e17976c5R1893
-          ./python-cryptography-Cherry-pick-fix-for-CVE-2023-49083-on-cryptography-40.patch
-        ];
-
-        # Tests would require overriding `cryptography-vectors`, which is not currently
-        # possible/desired, see: https://github.com/NixOS/nixpkgs/pull/281858#pullrequestreview-1841421866
-        doCheck = false;
-      });
+      cryptography = self.callPackage ./old-python-packages/cryptography.nix {};
 
       # This is the most recent version of `pyopenssl` that's still compatible with `cryptography` 40.
       # See https://github.com/NixOS/nixpkgs/pull/281858#issuecomment-1899358602
@@ -249,6 +223,14 @@ let
         ];
       });
 
+
+      fastapi = super.fastapi.overridePythonAttrs (old: rec {
+        # Flaky test:
+        #     ResourceWarning: Unclosed <MemoryObjectSendStream>
+        # Unclear whether it's flaky in general or only in this overridden package set.
+        doCheck = false;
+      });
+
       # Ceph does not support `kubernetes` >= 19, see:
       #     https://github.com/NixOS/nixpkgs/pull/281858#issuecomment-1900324090
       kubernetes = super.kubernetes.overridePythonAttrs (old: rec {
@@ -500,6 +482,7 @@ in rec {
 
     passthru = {
       inherit version;
+      inherit python; # to be able to test our overridden packages above individually with `nix-build -A`
       tests = {
         inherit (nixosTests)
           ceph-multi-node

pkgs/tools/filesystems/ceph/old-python-packages/cryptography-vectors.nix

@@ -0,0 +1,36 @@
+# This older version only exists because `ceph` needs it, see `cryptography.nix`.
+{
+  buildPythonPackage,
+  fetchPypi,
+  lib,
+  cryptography,
+}:
+
+buildPythonPackage rec {
+  pname = "cryptography-vectors";
+  # The test vectors must have the same version as the cryptography package
+  inherit (cryptography) version;
+  format = "setuptools";
+
+  src = fetchPypi {
+    pname = "cryptography_vectors";
+    inherit version;
+    hash = "sha256-hGBwa1tdDOSoVXHKM4nPiPcAu2oMYTPcn+D1ovW9oEE=";
+  };
+
+  # No tests included
+  doCheck = false;
+
+  pythonImportsCheck = [ "cryptography_vectors" ];
+
+  meta = with lib; {
+    description = "Test vectors for the cryptography package";
+    homepage = "https://cryptography.io/en/latest/development/test-vectors/";
+    # Source: https://github.com/pyca/cryptography/tree/master/vectors;
+    license = with licenses; [
+      asl20
+      bsd3
+    ];
+    maintainers = with maintainers; [ nh2 ];
+  };
+}

pkgs/tools/filesystems/ceph/old-python-packages/cryptography.nix

@@ -0,0 +1,135 @@
+# This older version only exists because `ceph` needs it, see its package.
+{
+  lib,
+  stdenv,
+  callPackage,
+  buildPythonPackage,
+  fetchPypi,
+  fetchpatch,
+  rustPlatform,
+  cargo,
+  rustc,
+  setuptoolsRustBuildHook,
+  openssl,
+  Security ? null,
+  isPyPy,
+  cffi,
+  pkg-config,
+  pytestCheckHook,
+  pytest-subtests,
+  pythonOlder,
+  pretend,
+  libiconv,
+  libxcrypt,
+  iso8601,
+  py,
+  pytz,
+  hypothesis,
+}:
+
+let
+  cryptography-vectors = callPackage ./cryptography-vectors.nix { };
+in
+buildPythonPackage rec {
+  pname = "cryptography";
+  version = "40.0.1"; # Also update the hash in vectors.nix
+  format = "setuptools";
+  disabled = pythonOlder "3.6";
+
+  src = fetchPypi {
+    inherit pname version;
+    hash = "sha256-KAPy+LHpX2FEGZJsfm9V2CivxhTKXtYVQ4d65mjMNHI=";
+  };
+
+  cargoDeps = rustPlatform.fetchCargoTarball {
+    inherit src;
+    sourceRoot = "${pname}-${version}/${cargoRoot}";
+    name = "${pname}-${version}";
+    hash = "sha256-gFfDTc2QWBWHBCycVH1dYlCsWQMVcRZfOBIau+njtDU=";
+  };
+
+  # Since Cryptography v40 is quite outdated, we need to backport
+  # security fixes that are only available in newer versions.
+  patches = [
+    # Fix https://nvd.nist.gov/vuln/detail/CVE-2023-49083 which has no upstream backport.
+    # See https://github.com/pyca/cryptography/commit/f09c261ca10a31fe41b1262306db7f8f1da0e48a#diff-f5134bf8f3cf0a5cc8601df55e50697acc866c603a38caff98802bd8e17976c5R1893
+    ./python-cryptography-Cherry-pick-fix-for-CVE-2023-49083-on-cryptography-40.patch
+
+    # Fix https://nvd.nist.gov/vuln/detail/CVE-2024-26130
+    # See https://github.com/pyca/cryptography/commit/97d231672763cdb5959a3b191e692a362f1b9e55
+    (fetchpatch {
+      name = "python-cryptography-CVE-2024-26130-dont-crash-when-a-PKCS-12-key-and-cert-dont-match-mmap-mode.patch";
+      url = "https://github.com/pyca/cryptography/commit/97d231672763cdb5959a3b191e692a362f1b9e55.patch";
+      hash = "sha256-l45NOzOWhHW4nY4OIRpdjYQRvUW8BROGWdpkAtvVn0Y=";
+    })
+  ];
+
+  postPatch = ''
+    substituteInPlace pyproject.toml \
+      --replace "--benchmark-disable" ""
+  '';
+
+  cargoRoot = "src/rust";
+
+  nativeBuildInputs = [
+    rustPlatform.cargoSetupHook
+    setuptoolsRustBuildHook
+    cargo
+    rustc
+    pkg-config
+  ] ++ lib.optionals (!isPyPy) [ cffi ];
+
+  buildInputs =
+    [ openssl ]
+    ++ lib.optionals stdenv.isDarwin [
+      Security
+      libiconv
+    ]
+    ++ lib.optionals (pythonOlder "3.9") [ libxcrypt ];
+
+  propagatedBuildInputs = lib.optionals (!isPyPy) [ cffi ];
+
+  nativeCheckInputs = [
+    cryptography-vectors
+    hypothesis
+    iso8601
+    pretend
+    py
+    pytestCheckHook
+    pytest-subtests
+    pytz
+  ];
+
+  pytestFlagsArray = [ "--disable-pytest-warnings" ];
+
+  disabledTestPaths =
+    [
+      # save compute time by not running benchmarks
+      "tests/bench"
+    ]
+    ++ lib.optionals (stdenv.isDarwin && stdenv.isAarch64) [
+      # aarch64-darwin forbids W+X memory, but this tests depends on it:
+      # * https://cffi.readthedocs.io/en/latest/using.html#callbacks
+      "tests/hazmat/backends/test_openssl_memleak.py"
+    ];
+
+  meta = with lib; {
+    description = "A package which provides cryptographic recipes and primitives";
+    longDescription = ''
+      Cryptography includes both high level recipes and low level interfaces to
+      common cryptographic algorithms such as symmetric ciphers, message
+      digests, and key derivation functions.
+      Our goal is for it to be your "cryptographic standard library". It
+      supports Python 2.7, Python 3.5+, and PyPy 5.4+.
+    '';
+    homepage = "https://github.com/pyca/cryptography";
+    changelog =
+      "https://cryptography.io/en/latest/changelog/#v" + replaceStrings [ "." ] [ "-" ] version;
+    license = with licenses; [
+      asl20
+      bsd3
+      psfl
+    ];
+    maintainers = with maintainers; [ nh2 ];
+  };
+}