From 7ee755683d0121ef9e701b4d1fdc28473002fa41 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Niklas=20Hamb=C3=BCchen?= Date: Sat, 28 Sep 2024 01:48:15 +0000 Subject: [PATCH] ceph: 18.2.4 -> 19.2.0 No-longer-necessary patches removed: * dmcrypt: https://github.com/ceph/ceph/pull/57925 * Finduring.cmake: https://github.com/ceph/ceph/commit/670118abfb9ef6135972a56514173e72b64ef8f4 * Python "packaging": https://github.com/ceph/ceph/commit/d1d8df6fafbb744d50787d3ffcd5c572397e421f Additional changes: * Add more runtime deps mentioned by upstream debian/control file. * Tell Ceph to use our specified Python dependencies, instead of the bundled ones (for `cephadm`). --- pkgs/tools/filesystems/ceph/default.nix | 82 ++++++++++++------------- 1 file changed, 40 insertions(+), 42 deletions(-) diff --git a/pkgs/tools/filesystems/ceph/default.nix b/pkgs/tools/filesystems/ceph/default.nix index e5beaf3841df..25075f612df8 100644 --- a/pkgs/tools/filesystems/ceph/default.nix +++ b/pkgs/tools/filesystems/ceph/default.nix @@ -7,10 +7,13 @@ , fetchpatch # Build time +, autoconf +, automake , cmake , ensureNewerSourcesHook , fmt , git +, libtool , makeWrapper , nasm , pkg-config @@ -22,10 +25,11 @@ # Runtime dependencies , arrow-cpp , babeltrace -, boost179 +, boost182 # using the version installed by ceph's `install-deps.sh` , bzip2 , cryptsetup , cunit +, e2fsprogs , doxygen , gperf , graphviz @@ -37,17 +41,22 @@ , libcap_ng , libnl , libxml2 +, lmdb , lttng-ust , lua +, lvm2 , lz4 , oath-toolkit , openldap -, python311 +, parted +, python311 # to get an idea which Python versions are supported by Ceph, see upstream `do_cmake.sh` (see `PYBUILD=` variable) , rdkafka , rocksdb , snappy +, openssh , sqlite , utf8proc +, xfsprogs , zlib , zstd @@ -247,7 +256,7 @@ let }; }; - boost = boost179.override { + boost = boost182.override { enablePython = true; inherit python; }; @@ -282,6 +291,9 @@ let virtualenv werkzeug + # src/cephadm/zipapp-reqs.txt + markupsafe + # src/pybind/mgr/requirements-required.txt cryptography jsonpatch @@ -292,54 +304,24 @@ let ]); inherit (ceph-python-env.python) sitePackages; - version = "18.2.4"; + version = "19.2.0"; src = fetchurl { url = "https://download.ceph.com/tarballs/ceph-${version}.tar.gz"; - hash = "sha256-EFqteP3Jo+hASXVesH6gkjDjFO7/1RN151tIf/lQ06s="; + hash = "sha256-30vkW1j49hFIxyxzkssSKVSq0VqiwLfDtOb62xfxadM="; }; in rec { ceph = stdenv.mkDerivation { pname = "ceph"; inherit src version; - patches = [ - # Fixes mgr not being able to import `packaging` due to autotools >= 70. - # Remove once https://github.com/ceph/ceph/pull/58624 is merged, see - # https://github.com/NixOS/nixpkgs/pull/330226#issuecomment-2268421031 - (fetchpatch { - url = "https://github.com/ceph/ceph/commit/8da2d857fa8fdfedd7aad0ca90e1780a3ed085c9.patch"; - name = "ceph-mgr-python-fix-packaging-import.patch"; - hash = "sha256-3Yl1X6UfTf0XCXJxgRnM/Js9sz8tS+hsqViY6gDExoI="; - }) - - # Fixes cryptesetup version parsing regex, see - # * https://github.com/NixOS/nixpkgs/issues/334227 - # * https://www.mail-archive.com/ceph-users@ceph.io/msg26309.html - # * https://github.com/ceph/ceph/pull/58997 - # Remove once we're on the next version of Ceph 18, when this should be in: - # https://github.com/ceph/ceph/pull/58997 - (fetchpatch { - url = "https://github.com/ceph/ceph/commit/6ae874902b63652fa199563b6e7950cd75151304.patch"; - name = "ceph-reef-ceph-volume-fix-set_dmcrypt_no_workqueue.patch"; - hash = "sha256-r+7hcCz2WF/rJfgKwTatKY9unJlE8Uw3fmOyaY5jVH0="; - }) - (fetchpatch { - url = "https://github.com/ceph/ceph/commit/607eb34b2c278566c386efcbf3018629cf08ccfd.patch"; - name = "ceph-volume-fix-set_dmcrypt_no_workqueue-regex.patch"; - hash = "sha256-q28Q7OIyFoMyMBCPXGA+AdNqp+9/6J/XwD4ODjx+JXY="; - }) - ]; - - postPatch = '' - substituteInPlace cmake/modules/Finduring.cmake \ - --replace-fail "liburing.a liburing" "uring" - ''; - nativeBuildInputs = [ + autoconf # `autoreconf` is called, e.g. for `qatlib_ext` + automake # `aclocal` is called, e.g. for `qatlib_ext` cmake fmt git makeWrapper + libtool # used e.g. for `qatlib_ext` nasm pkg-config python @@ -357,17 +339,23 @@ in rec { babeltrace boost bzip2 + # Adding `ceph-python-env` here adds the env's `site-packages` to `PYTHONPATH` during the build. + # This is important, otherwise the build system may not find the Python deps and then + # silently skip installing ceph-volume and other Ceph python tools. ceph-python-env cryptsetup cunit + e2fsprogs # according to `debian/control` file, `ceph-volume` is supposed to use it gperf gtest icu libcap libnl libxml2 + lmdb lttng-ust lua + lvm2 # according to `debian/control` file, e.g. `pvs` command used by `src/ceph-volume/ceph_volume/api/lvm.py` lz4 malloc oath-toolkit @@ -375,11 +363,14 @@ in rec { optLibatomic_ops optLibs3 optYasm + parted # according to `debian/control` file, used by `src/ceph-volume/ceph_volume/util/disk.py` rdkafka rocksdb' snappy + openssh # according to `debian/control` file, `ssh` command used by `cephadm` sqlite utf8proc + xfsprogs # according to `debian/control` file, `ceph-volume` is supposed to use it zlib zstd ] ++ lib.optionals stdenv.hostPlatform.isLinux [ @@ -402,7 +393,11 @@ in rec { optLibedit ]; - pythonPath = [ ceph-python-env "${placeholder "out"}/${ceph-python-env.sitePackages}" ]; + # Picked up, amongst others, by `wrapPythonPrograms`. + pythonPath = [ + ceph-python-env + "${placeholder "out"}/${ceph-python-env.sitePackages}" + ]; # replace /sbin and /bin based paths with direct nix store paths # increase the `command` buffer size since 2 nix store paths cannot fit within 128 characters @@ -413,9 +408,8 @@ in rec { --replace "/sbin/modprobe" "${kmod}/bin/modprobe" \ --replace "/bin/grep" "${gnugrep}/bin/grep" - # install target needs to be in PYTHONPATH for "*.pth support" check to succeed - # set PYTHONPATH, so the build system doesn't silently skip installing ceph-volume and others - export PYTHONPATH=${ceph-python-env}/${sitePackages}:$lib/${sitePackages}:$out/${sitePackages} + # The install target needs to be in PYTHONPATH for "*.pth support" check to succeed + export PYTHONPATH=$PYTHONPATH:$lib/${sitePackages}:$out/${sitePackages} patchShebangs src/ ''; @@ -453,6 +447,10 @@ in rec { "-DWITH_SYSTEM_UTF8PROC:BOOL=ON" "-DWITH_SYSTEM_ZSTD:BOOL=ON" + # Use our own python libraries too, see: + # https://github.com/NixOS/nixpkgs/pull/344993#issuecomment-2391046329 + "-DCEPHADM_BUNDLED_DEPENDENCIES=none" + # TODO breaks with sandbox, tries to download stuff with npm "-DWITH_MGR_DASHBOARD_FRONTEND:BOOL=OFF" # WITH_XFS has been set default ON from Ceph 16, keeping it optional in nixpkgs for now