ceph: 19.2.3 -> 20.2.1 (#494583)
This commit is contained in:
Niklas Hambüchen
@@ -245,6 +245,9 @@ See <https://github.com/NixOS/nixpkgs/issues/481673>.
|
||||
|
||||
- `programs.light` was removed from nixpkgs due to the corresponding package being unmaintained upstream. `brightnessctl` and `programs.acpilight` offer replacements.
|
||||
|
||||
- `ceph` has been upgraded to v20. See the [Ceph "tentacle" release notes](https://docs.ceph.com/en/latest/releases/tentacle/#v20-2-0-tentacle) for details and recommended upgrade procedure.
|
||||
Note that **upgrades of server-side components are one-way**, and downgrading e.g. an OSD from *Tentacle* to *Squid* is not just not supported but is known to break.
|
||||
|
||||
- The `networking.wireless` module has been security hardened by default: the `wpa_supplicant` daemon now runs under an unprivileged user with restricted access to the system.
|
||||
|
||||
As part of these changes, `/etc/wpa_supplicant.conf` has been deprecated: the NixOS-generated configuration file is now linked to `/etc/wpa_supplicant/nixos.conf` and `/etc/wpa_supplicant/imperative.conf` has been added for imperatively configuring `wpa_supplicant` or when using [allowAuxiliaryImperativeNetworks](#opt-networking.wireless.allowAuxiliaryImperativeNetworks).
|
||||
|
||||
@@ -43,6 +43,7 @@ let
|
||||
{ pkgs, ... }:
|
||||
{
|
||||
virtualisation = {
|
||||
memorySize = 2048;
|
||||
emptyDiskImages = [
|
||||
20480
|
||||
20480
|
||||
@@ -92,6 +93,10 @@ let
|
||||
cfg.osd2.name
|
||||
];
|
||||
};
|
||||
rgw = {
|
||||
enable = true;
|
||||
daemons = [ cfg.monA.name ];
|
||||
};
|
||||
};
|
||||
};
|
||||
|
||||
@@ -100,6 +105,8 @@ let
|
||||
# For other ways to deploy a ceph cluster, look at the documentation at
|
||||
# https://docs.ceph.com/docs/master/
|
||||
testScript = ''
|
||||
import json
|
||||
|
||||
start_all()
|
||||
|
||||
monA.wait_for_unit("network.target")
|
||||
@@ -194,6 +201,16 @@ let
|
||||
"ceph osd pool delete single-node-other-test single-node-other-test --yes-i-really-really-mean-it",
|
||||
)
|
||||
|
||||
# Bootstrap RGW
|
||||
monA.succeed(
|
||||
"sudo -u ceph mkdir -p /var/lib/ceph/radosgw/ceph-${cfg.monA.name}",
|
||||
"ceph auth get-or-create client.${cfg.monA.name} osd 'allow rwx' mon 'allow rw' > /var/lib/ceph/radosgw/ceph-${cfg.monA.name}/keyring",
|
||||
"chown ceph:ceph /var/lib/ceph/radosgw/ceph-${cfg.monA.name}/keyring",
|
||||
"systemctl start ceph-rgw-${cfg.monA.name}",
|
||||
)
|
||||
monA.wait_for_unit("ceph-rgw-${cfg.monA.name}")
|
||||
monA.wait_for_open_port(7480)
|
||||
|
||||
# Shut down ceph by stopping ceph.target.
|
||||
monA.succeed("systemctl stop ceph.target")
|
||||
|
||||
@@ -204,6 +221,7 @@ let
|
||||
monA.wait_for_unit("ceph-osd-${cfg.osd0.name}")
|
||||
monA.wait_for_unit("ceph-osd-${cfg.osd1.name}")
|
||||
monA.wait_for_unit("ceph-osd-${cfg.osd2.name}")
|
||||
monA.wait_for_unit("ceph-rgw-${cfg.monA.name}")
|
||||
|
||||
# Ensure the cluster comes back up again
|
||||
monA.succeed("ceph -s | grep 'mon: 1 daemons'")
|
||||
@@ -222,6 +240,32 @@ let
|
||||
monA.wait_for_open_port(8080)
|
||||
monA.wait_until_succeeds("curl -q --fail http://localhost:8080")
|
||||
monA.wait_until_succeeds("ceph -s | grep 'HEALTH_OK'")
|
||||
|
||||
# Initialize dashboard creds
|
||||
monA.succeed(
|
||||
"echo 'foo bar baz qux' > /tmp/dashboard_pw",
|
||||
"ceph dashboard ac-user-create admin -i /tmp/dashboard_pw administrator",
|
||||
"ceph dashboard set-rgw-credentials",
|
||||
)
|
||||
|
||||
# Get dashboard auth token
|
||||
auth_payload = json.dumps({"username": "admin", "password": "foo bar baz qux"})
|
||||
auth_response = json.loads(monA.succeed(
|
||||
f"curl --fail -s -X POST -H 'Accept: application/vnd.ceph.api.v1.0+json' -H 'Content-Type: application/json' -d '{auth_payload}' http://localhost:8080/api/auth",
|
||||
))
|
||||
token = auth_response["token"]
|
||||
|
||||
# Check cluster health via dashboard API
|
||||
health = json.loads(monA.succeed(
|
||||
f"curl --fail -s -H 'Accept: application/vnd.ceph.api.v1.0+json' -H 'Authorization: Bearer {token}' http://localhost:8080/api/health/minimal",
|
||||
))
|
||||
assert health["health"]["status"] == "HEALTH_OK"
|
||||
|
||||
# List daemons via REST API
|
||||
rgw_daemons = json.loads(monA.succeed(
|
||||
f"curl --fail -s -H 'Accept: application/vnd.ceph.api.v1.0+json' -H 'Authorization: Bearer {token}' http://localhost:8080/api/rgw/daemon",
|
||||
))
|
||||
assert rgw_daemons[0]["id"] == "a"
|
||||
'';
|
||||
in
|
||||
{
|
||||
|
||||
@@ -0,0 +1,75 @@
|
||||
# Ceph
|
||||
|
||||
Ceph is a Distributed storage system providing object based (rados and S3), block based (rbd and iscsi), and filesystem based (cephfs, NFS, samba) storage.
|
||||
|
||||
Because Ceph is used for massive storage clusters in the wild, any changes can lead to actual data loss.
|
||||
Besides having to handle changes – in particular breaking changes – somewhat carefully it also means that Ceph does move slow in terms of development.
|
||||
It should be stressed here that slow release pacing is not a bad thing!
|
||||
|
||||
If you are looking at this page because a change in, let's say Python packaging, broke the build of *qemu_full*, then if you are not actually using Ceph for anything[^krbd] it may be easier to mask out Ceph from your build[^mask].
|
||||
One common way that Ceph is being pulled in for users who do not actually use it is due to the `enableCephFS` flag of *samba4Full*.
|
||||
|
||||
[^krbd]: Or are only using implicit *krbd* which lives in the kernel and thus doesn't require this package.
|
||||
[^mask]: You can achieve this using several ways, including an overlay setting *ceph* to *null*, or something like `qemu_full.override { cephSupport = false; }`.
|
||||
|
||||
## Patches
|
||||
|
||||
Historically Ceph often needed patches to be compatible with newer versions of *gcc*, Python, and other related software.
|
||||
On occasion there may also be patches which need to be backported from the development of the current release as they may have been merged, but no new release has been made including those changes.
|
||||
Because Ceph uses embedded Python interpreters in some daemons these patches need to be applied to multiple builds.
|
||||
To keep drift from happening and make this more visible [the source code itself is being patched](./src.nix) before passing it on as *src* for other derivations.
|
||||
|
||||
## Scope
|
||||
|
||||
This package often requires patches, or vendored packaging so we can keep an older version of a library around specifically for Ceph.
|
||||
For this reason Ceph is being maintained [within its own scope](./scope.nix), which allows us to inject such vendored packages more easily.
|
||||
An example at time of writing would be *RocksDB* which is the storage format used by the Ceph storage daemons (OSDs) to store data on block devices directly.
|
||||
|
||||
If for whatever reason you want or need to patch Ceph, you can use the *passthru* attribute `overrideScope` to get access to the full Ceph scope.
|
||||
For instance an overlay can be used to override dependencies as well as patches.
|
||||
|
||||
<details><summary>Example overlay for overriding Ceph</summary>
|
||||
|
||||
```nix
|
||||
globalFinal: globalPrev: {
|
||||
ceph = builtins.getAttr "ceph" (
|
||||
globalPrev.ceph.overrideScope (
|
||||
final: prev: {
|
||||
# `prev.rocksdb` here is the vendored version of rocksdb
|
||||
# this means you can apply patches to the otherwise hidden packages
|
||||
|
||||
ceph-src = prev.ceph-src.overrideAttrs (
|
||||
{
|
||||
patches ? [ ],
|
||||
...
|
||||
}:
|
||||
{
|
||||
patches = patches ++ [
|
||||
# your patch goes here
|
||||
(globalFinal.fetchurl {
|
||||
# ...
|
||||
})
|
||||
];
|
||||
}
|
||||
);
|
||||
}
|
||||
)
|
||||
);
|
||||
}
|
||||
```
|
||||
|
||||
</details>
|
||||
|
||||
## Python
|
||||
|
||||
Ceph is very intertwined with Python due to the subinterpreters.
|
||||
The [*nixpkgs* Ceph PyO3 tracking issue](https://github.com/NixOS/nixpkgs/issues/380823) has some information on this and during the [*nixpkgs* Ceph 20.2.1 merge](https://github.com/NixOS/nixpkgs/pull/494583) several issues surfaced.
|
||||
When using Ceph packaged via *nixpkgs* you should not run into PyO3 issues, if you do, please file a bug report.
|
||||
|
||||
There are two MGR modules which do attempt to load libraries incompatible with PyO3; *cephadm* and *diskprediction_local*.
|
||||
*cephadm* conceptually does not work on NixOS since NixOS intentionally makes *systemd* configuration read-only.
|
||||
However the *cephadm* MGR module is enabled by default, and it is not been patched to be less prone to PyO3 issues.
|
||||
Similarly *diskprediction_local* will attempt to load Numpy/SciPy, leading to PyO3 errors.
|
||||
These errors are not fatal, however they do render the modules unusable.
|
||||
You can disable the modules using `ceph mgr module disable` to silence the errors.
|
||||
|
||||
@@ -1,353 +0,0 @@
|
||||
# This is is arrow-cpp < 20 used as a workaround for
|
||||
# Ceph not supporting >= yet, taken from nixpkgs commit
|
||||
# 97ae53798f6a7c7c3c259ad8c2cbcede6ca34b2a~
|
||||
# This should be entirely removed when upstream bug
|
||||
# https://tracker.ceph.com/issues/71269
|
||||
# is fixed.
|
||||
{
|
||||
stdenv,
|
||||
lib,
|
||||
fetchurl,
|
||||
fetchpatch2,
|
||||
fetchFromGitHub,
|
||||
fixDarwinDylibNames,
|
||||
autoconf,
|
||||
aws-sdk-cpp,
|
||||
aws-sdk-cpp-arrow ? aws-sdk-cpp.override {
|
||||
apis = [
|
||||
"cognito-identity"
|
||||
"config"
|
||||
"identity-management"
|
||||
"s3"
|
||||
"sts"
|
||||
"transfer"
|
||||
];
|
||||
},
|
||||
boost,
|
||||
brotli,
|
||||
bzip2,
|
||||
cmake,
|
||||
crc32c,
|
||||
curl,
|
||||
flatbuffers,
|
||||
gflags,
|
||||
glog,
|
||||
google-cloud-cpp,
|
||||
grpc,
|
||||
gtest,
|
||||
libbacktrace,
|
||||
lz4,
|
||||
minio,
|
||||
ninja,
|
||||
nlohmann_json,
|
||||
openssl,
|
||||
perl,
|
||||
pkg-config,
|
||||
protobuf_32,
|
||||
python3,
|
||||
rapidjson,
|
||||
re2,
|
||||
snappy,
|
||||
sqlite,
|
||||
thrift,
|
||||
tzdata,
|
||||
utf8proc,
|
||||
which,
|
||||
zlib,
|
||||
zstd,
|
||||
testers,
|
||||
enableShared ? !stdenv.hostPlatform.isStatic,
|
||||
enableFlight ? stdenv.buildPlatform == stdenv.hostPlatform,
|
||||
# Disable also on RiscV
|
||||
# configure: error: cannot determine number of significant virtual address bits
|
||||
enableJemalloc ?
|
||||
!stdenv.hostPlatform.isDarwin && !stdenv.hostPlatform.isAarch64 && !stdenv.hostPlatform.isRiscV64,
|
||||
enableS3 ? true,
|
||||
# google-cloud-cpp fails to build on RiscV
|
||||
enableGcs ? !stdenv.hostPlatform.isDarwin && !stdenv.hostPlatform.isRiscV64,
|
||||
}:
|
||||
|
||||
let
|
||||
arrow-testing = fetchFromGitHub {
|
||||
name = "arrow-testing";
|
||||
owner = "apache";
|
||||
repo = "arrow-testing";
|
||||
rev = "4d209492d514c2d3cb2d392681b9aa00e6d8da1c";
|
||||
hash = "sha256-IkiCbuy0bWyClPZ4ZEdkEP7jFYLhM7RCuNLd6Lazd4o=";
|
||||
};
|
||||
|
||||
parquet-testing = fetchFromGitHub {
|
||||
name = "parquet-testing";
|
||||
owner = "apache";
|
||||
repo = "parquet-testing";
|
||||
rev = "c7cf1374cf284c0c73024cd1437becea75558bf8";
|
||||
hash = "sha256-DThjyZ34LajHwXZy1IhYKUGUG/ejQ9WvBNuI8eUKmSs=";
|
||||
};
|
||||
|
||||
version = "19.0.1";
|
||||
in
|
||||
stdenv.mkDerivation (finalAttrs: {
|
||||
pname = "arrow-cpp";
|
||||
inherit version;
|
||||
|
||||
src = fetchFromGitHub {
|
||||
owner = "apache";
|
||||
repo = "arrow";
|
||||
rev = "apache-arrow-${version}";
|
||||
hash = "sha256-toHwUIOZRpgR0K7pQtT5nqWpO9G7AuHYTcvA6UVg9lA=";
|
||||
};
|
||||
|
||||
sourceRoot = "${finalAttrs.src.name}/cpp";
|
||||
|
||||
patches = [
|
||||
(fetchpatch2 {
|
||||
name = "protobuf-30-compat.patch";
|
||||
url = "https://github.com/apache/arrow/pull/46136.patch";
|
||||
hash = "sha256-WTpe/eT3himlCHN/R78w1sF0HG859mE2ZN70U+9N8Ag=";
|
||||
stripLen = 1;
|
||||
})
|
||||
(fetchpatch2 {
|
||||
name = "cmake-fix.patch";
|
||||
url = "https://github.com/apache/arrow/commit/48c0bbbd4a2eedcca518caeb7f7547c7988dc740.patch?full_index=1";
|
||||
hash = "sha256-i/vZy/61VYP+mo1AxfoiBSjTip04vhFOh3hGjHCJy6g=";
|
||||
stripLen = 1; # applying patch from within `cpp/` subdirectory
|
||||
})
|
||||
];
|
||||
|
||||
# versions are all taken from
|
||||
# https://github.com/apache/arrow/blob/apache-arrow-${version}/cpp/thirdparty/versions.txt
|
||||
|
||||
env =
|
||||
lib.optionalAttrs enableJemalloc {
|
||||
# jemalloc: arrow uses a custom prefix to prevent default allocator symbol
|
||||
# collisions as well as custom build flags
|
||||
ARROW_JEMALLOC_URL = fetchurl {
|
||||
url = "https://github.com/jemalloc/jemalloc/releases/download/5.3.0/jemalloc-5.3.0.tar.bz2";
|
||||
hash = "sha256-LbgtHnEZ3z5xt2QCGbbf6EeJvAU3mDw7esT3GJrs/qo=";
|
||||
};
|
||||
}
|
||||
// {
|
||||
# mimalloc: arrow uses custom build flags for mimalloc
|
||||
ARROW_MIMALLOC_URL = fetchFromGitHub {
|
||||
owner = "microsoft";
|
||||
repo = "mimalloc";
|
||||
rev = "v2.0.6";
|
||||
hash = "sha256-u2ITXABBN/dwU+mCIbL3tN1f4c17aBuSdNTV+Adtohc=";
|
||||
};
|
||||
|
||||
ARROW_XSIMD_URL = fetchFromGitHub {
|
||||
owner = "xtensor-stack";
|
||||
repo = "xsimd";
|
||||
rev = "13.0.0";
|
||||
hash = "sha256-qElJYW5QDj3s59L3NgZj5zkhnUMzIP2mBa1sPks3/CE=";
|
||||
};
|
||||
|
||||
ARROW_SUBSTRAIT_URL = fetchFromGitHub {
|
||||
owner = "substrait-io";
|
||||
repo = "substrait";
|
||||
rev = "v0.44.0";
|
||||
hash = "sha256-V739IFTGPtbGPlxcOi8sAaYSDhNUEpITvN9IqdPReug=";
|
||||
};
|
||||
}
|
||||
// lib.optionalAttrs finalAttrs.doInstallCheck {
|
||||
ARROW_TEST_DATA = "${arrow-testing}/data";
|
||||
PARQUET_TEST_DATA = "${parquet-testing}/data";
|
||||
GTEST_FILTER =
|
||||
let
|
||||
# Upstream Issue: https://issues.apache.org/jira/browse/ARROW-11398
|
||||
filteredTests =
|
||||
lib.optionals stdenv.hostPlatform.isAarch64 [
|
||||
"TestFilterKernelWithNumeric/3.CompareArrayAndFilterRandomNumeric"
|
||||
"TestFilterKernelWithNumeric/7.CompareArrayAndFilterRandomNumeric"
|
||||
"TestCompareKernel.PrimitiveRandomTests"
|
||||
]
|
||||
++ lib.optionals enableS3 [
|
||||
"S3OptionsTest.FromUri"
|
||||
"S3RegionResolutionTest.NonExistentBucket"
|
||||
"S3RegionResolutionTest.PublicBucket"
|
||||
"S3RegionResolutionTest.RestrictedBucket"
|
||||
"TestMinioServer.Connect"
|
||||
"TestS3FS.*"
|
||||
"TestS3FSGeneric.*"
|
||||
]
|
||||
++ lib.optionals stdenv.hostPlatform.isDarwin [
|
||||
# TODO: revisit at 12.0.0 or when
|
||||
# https://github.com/apache/arrow/commit/295c6644ca6b67c95a662410b2c7faea0920c989
|
||||
# is available, see
|
||||
# https://github.com/apache/arrow/pull/15288#discussion_r1071244661
|
||||
"ExecPlanExecution.StressSourceSinkStopped"
|
||||
];
|
||||
in
|
||||
"-${lib.concatStringsSep ":" filteredTests}";
|
||||
};
|
||||
|
||||
nativeBuildInputs = [
|
||||
cmake
|
||||
pkg-config
|
||||
ninja
|
||||
autoconf # for vendored jemalloc
|
||||
flatbuffers
|
||||
]
|
||||
++ lib.optional stdenv.hostPlatform.isDarwin fixDarwinDylibNames;
|
||||
buildInputs = [
|
||||
boost
|
||||
brotli
|
||||
bzip2
|
||||
flatbuffers
|
||||
gflags
|
||||
glog
|
||||
gtest
|
||||
libbacktrace
|
||||
lz4
|
||||
nlohmann_json # alternative JSON parser to rapidjson
|
||||
protobuf_32 # substrait requires protobuf
|
||||
rapidjson
|
||||
re2
|
||||
snappy
|
||||
thrift
|
||||
utf8proc
|
||||
zlib
|
||||
zstd
|
||||
]
|
||||
++ lib.optionals enableFlight [
|
||||
grpc
|
||||
openssl
|
||||
sqlite
|
||||
]
|
||||
++ lib.optionals enableS3 [
|
||||
aws-sdk-cpp-arrow
|
||||
openssl
|
||||
]
|
||||
++ lib.optionals enableGcs [
|
||||
crc32c
|
||||
curl
|
||||
google-cloud-cpp
|
||||
grpc
|
||||
nlohmann_json
|
||||
];
|
||||
|
||||
preConfigure = ''
|
||||
patchShebangs build-support/
|
||||
substituteInPlace "src/arrow/vendored/datetime/tz.cpp" \
|
||||
--replace-fail 'discover_tz_dir();' '"${tzdata}/share/zoneinfo";'
|
||||
'';
|
||||
|
||||
cmakeFlags = [
|
||||
"-DCMAKE_FIND_PACKAGE_PREFER_CONFIG=ON"
|
||||
"-DARROW_BUILD_SHARED=${if enableShared then "ON" else "OFF"}"
|
||||
"-DARROW_BUILD_STATIC=${if enableShared then "OFF" else "ON"}"
|
||||
"-DARROW_BUILD_TESTS=${if enableShared then "ON" else "OFF"}"
|
||||
"-DARROW_BUILD_INTEGRATION=ON"
|
||||
"-DARROW_BUILD_UTILITIES=ON"
|
||||
"-DARROW_EXTRA_ERROR_CONTEXT=ON"
|
||||
"-DARROW_VERBOSE_THIRDPARTY_BUILD=ON"
|
||||
"-DARROW_DEPENDENCY_SOURCE=SYSTEM"
|
||||
"-Dxsimd_SOURCE=AUTO"
|
||||
"-DARROW_DEPENDENCY_USE_SHARED=${if enableShared then "ON" else "OFF"}"
|
||||
"-DARROW_COMPUTE=ON"
|
||||
"-DARROW_CSV=ON"
|
||||
"-DARROW_DATASET=ON"
|
||||
"-DARROW_FILESYSTEM=ON"
|
||||
"-DARROW_FLIGHT_SQL=${if enableFlight then "ON" else "OFF"}"
|
||||
"-DARROW_HDFS=ON"
|
||||
"-DARROW_IPC=ON"
|
||||
"-DARROW_JEMALLOC=${if enableJemalloc then "ON" else "OFF"}"
|
||||
"-DARROW_JSON=ON"
|
||||
"-DARROW_USE_GLOG=ON"
|
||||
"-DARROW_WITH_BACKTRACE=ON"
|
||||
"-DARROW_WITH_BROTLI=ON"
|
||||
"-DARROW_WITH_BZ2=ON"
|
||||
"-DARROW_WITH_LZ4=ON"
|
||||
"-DARROW_WITH_NLOHMANN_JSON=ON"
|
||||
"-DARROW_WITH_SNAPPY=ON"
|
||||
"-DARROW_WITH_UTF8PROC=ON"
|
||||
"-DARROW_WITH_ZLIB=ON"
|
||||
"-DARROW_WITH_ZSTD=ON"
|
||||
"-DARROW_MIMALLOC=ON"
|
||||
"-DARROW_SUBSTRAIT=ON"
|
||||
"-DARROW_FLIGHT=${if enableFlight then "ON" else "OFF"}"
|
||||
"-DARROW_FLIGHT_TESTING=${if enableFlight then "ON" else "OFF"}"
|
||||
"-DARROW_S3=${if enableS3 then "ON" else "OFF"}"
|
||||
"-DARROW_GCS=${if enableGcs then "ON" else "OFF"}"
|
||||
# Parquet options:
|
||||
"-DARROW_PARQUET=ON"
|
||||
"-DPARQUET_BUILD_EXECUTABLES=ON"
|
||||
"-DPARQUET_REQUIRE_ENCRYPTION=ON"
|
||||
]
|
||||
++ lib.optionals (!enableShared) [ "-DARROW_TEST_LINKAGE=static" ]
|
||||
++ lib.optionals stdenv.hostPlatform.isDarwin [
|
||||
"-DCMAKE_INSTALL_RPATH=@loader_path/../lib" # needed for tools executables
|
||||
]
|
||||
++ lib.optionals (!stdenv.hostPlatform.isx86_64) [ "-DARROW_USE_SIMD=OFF" ]
|
||||
++ lib.optionals enableS3 [
|
||||
"-DAWSSDK_CORE_HEADER_FILE=${aws-sdk-cpp-arrow}/include/aws/core/Aws.h"
|
||||
];
|
||||
|
||||
doInstallCheck = true;
|
||||
|
||||
__darwinAllowLocalNetworking = true;
|
||||
|
||||
nativeInstallCheckInputs = [
|
||||
perl
|
||||
which
|
||||
sqlite
|
||||
]
|
||||
++ lib.optionals enableS3 [ minio ]
|
||||
++ lib.optionals enableFlight [ python3 ];
|
||||
|
||||
installCheckPhase =
|
||||
let
|
||||
disabledTests = [
|
||||
# flaky
|
||||
"arrow-flight-test"
|
||||
# requires networking
|
||||
"arrow-gcsfs-test"
|
||||
"arrow-flight-integration-test"
|
||||
];
|
||||
in
|
||||
''
|
||||
runHook preInstallCheck
|
||||
|
||||
ctest -L unittest --exclude-regex '^(${lib.concatStringsSep "|" disabledTests})$'
|
||||
|
||||
runHook postInstallCheck
|
||||
'';
|
||||
|
||||
__structuredAttrs = true;
|
||||
|
||||
meta = {
|
||||
description = "Cross-language development platform for in-memory data";
|
||||
homepage = "https://arrow.apache.org/docs/cpp/";
|
||||
license = lib.licenses.asl20;
|
||||
platforms = lib.platforms.unix;
|
||||
maintainers = with lib.maintainers; [
|
||||
tobim
|
||||
veprbl
|
||||
cpcloud
|
||||
];
|
||||
pkgConfigModules = [
|
||||
"arrow"
|
||||
"arrow-acero"
|
||||
"arrow-compute"
|
||||
"arrow-csv"
|
||||
"arrow-dataset"
|
||||
"arrow-filesystem"
|
||||
"arrow-flight"
|
||||
"arrow-flight-sql"
|
||||
"arrow-flight-testing"
|
||||
"arrow-json"
|
||||
"arrow-substrait"
|
||||
"arrow-testing"
|
||||
"parquet"
|
||||
];
|
||||
};
|
||||
passthru = {
|
||||
inherit
|
||||
enableFlight
|
||||
enableJemalloc
|
||||
enableS3
|
||||
enableGcs
|
||||
;
|
||||
tests.pkg-config = testers.testMetaPkgConfig finalAttrs.finalPackage;
|
||||
};
|
||||
})
|
||||
@@ -1,69 +0,0 @@
|
||||
Backported from <https://github.com/ceph/ceph/commit/857eedbe6c9ed80ed0625bd0aa27b1a1e85f8d59>.
|
||||
|
||||
Original author: Adam Emerson <aemerson@redhat.com>
|
||||
|
||||
diff --git a/CMakeLists.txt b/CMakeLists.txt
|
||||
index bbd63a6a006..bbd7c737feb 100644
|
||||
--- a/CMakeLists.txt
|
||||
+++ b/CMakeLists.txt
|
||||
@@ -666,7 +666,7 @@ option(WITH_SYSTEM_BOOST "require and build with system Boost" OFF)
|
||||
# Boost::thread depends on Boost::atomic, so list it explicitly.
|
||||
set(BOOST_COMPONENTS
|
||||
atomic chrono thread system regex random program_options date_time
|
||||
- iostreams context coroutine)
|
||||
+ iostreams context coroutine url)
|
||||
set(BOOST_HEADER_COMPONENTS container)
|
||||
|
||||
if(WITH_MGR)
|
||||
diff --git a/src/mds/BoostUrlImpl.cc b/src/mds/BoostUrlImpl.cc
|
||||
deleted file mode 100644
|
||||
index 479f4c6d75d..00000000000
|
||||
--- a/src/mds/BoostUrlImpl.cc
|
||||
+++ /dev/null
|
||||
@@ -1,8 +0,0 @@
|
||||
-/*
|
||||
- * https://www.boost.org/doc/libs/1_82_0/libs/url/doc/html/url/overview.html#url.overview.requirements
|
||||
- *
|
||||
- * To use the library as header-only; that is, to eliminate the requirement
|
||||
- * to link a program to a static or dynamic Boost.URL library,
|
||||
- * simply place the following line in exactly one source file in your project.
|
||||
- */
|
||||
-#include <boost/url/src.hpp>
|
||||
diff --git a/src/mds/CMakeLists.txt b/src/mds/CMakeLists.txt
|
||||
index 0c6c31a3c51..5c98db76e4d 100644
|
||||
--- a/src/mds/CMakeLists.txt
|
||||
+++ b/src/mds/CMakeLists.txt
|
||||
@@ -45,12 +45,12 @@ set(mds_srcs
|
||||
QuiesceDbManager.cc
|
||||
QuiesceAgent.cc
|
||||
MDSRankQuiesce.cc
|
||||
- BoostUrlImpl.cc
|
||||
${CMAKE_SOURCE_DIR}/src/common/TrackedOp.cc
|
||||
${CMAKE_SOURCE_DIR}/src/common/MemoryModel.cc
|
||||
${CMAKE_SOURCE_DIR}/src/osdc/Journaler.cc
|
||||
${CMAKE_SOURCE_DIR}/src/mgr/MDSPerfMetricTypes.cc)
|
||||
add_library(mds STATIC ${mds_srcs})
|
||||
target_link_libraries(mds PRIVATE
|
||||
+ Boost::url
|
||||
heap_profiler cpu_profiler osdc ${LUA_LIBRARIES})
|
||||
target_include_directories(mds PRIVATE "${LUA_INCLUDE_DIR}")
|
||||
diff --git a/src/test/mds/CMakeLists.txt b/src/test/mds/CMakeLists.txt
|
||||
index f80abe75083..18ebb648e68 100644
|
||||
--- a/src/test/mds/CMakeLists.txt
|
||||
+++ b/src/test/mds/CMakeLists.txt
|
||||
@@ -18,11 +18,10 @@ target_link_libraries(unittest_mds_sessionfilter mds osdc ceph-common global ${B
|
||||
add_executable(unittest_mds_quiesce_db
|
||||
TestQuiesceDb.cc
|
||||
../../../src/mds/QuiesceDbManager.cc
|
||||
- ../../../src/mds/BoostUrlImpl.cc
|
||||
$<TARGET_OBJECTS:unit-main>
|
||||
)
|
||||
add_ceph_unittest(unittest_mds_quiesce_db)
|
||||
-target_link_libraries(unittest_mds_quiesce_db ceph-common global)
|
||||
+target_link_libraries(unittest_mds_quiesce_db ceph-common global Boost::url)
|
||||
|
||||
# unittest_mds_quiesce_agent
|
||||
add_executable(unittest_mds_quiesce_agent
|
||||
--
|
||||
2.47.0
|
||||
|
||||
@@ -1,20 +0,0 @@
|
||||
Excerpted from <https://aur.archlinux.org/cgit/aur.git/commit/?h=ceph&id=8c5cc7d8deec002f7596b6d0860859a0a718f12b>.
|
||||
|
||||
Original author: Bazaah <github@luxolus.com>
|
||||
|
||||
diff --git a/src/mgr/PyModule.cc b/src/mgr/PyModule.cc
|
||||
index 084cf3ffc1e..010a1177a88 100644
|
||||
--- a/src/mgr/PyModule.cc
|
||||
+++ b/src/mgr/PyModule.cc
|
||||
@@ -36,6 +36,11 @@ std::string PyModule::mgr_store_prefix = "mgr/";
|
||||
|
||||
// Courtesy of http://stackoverflow.com/questions/1418015/how-to-get-python-exception-text
|
||||
#define BOOST_BIND_GLOBAL_PLACEHOLDERS
|
||||
+// Fix instances of "'BOOST_PP_ITERATION_02' was not declared in this scope; did you mean 'BOOST_PP_ITERATION_05'"
|
||||
+// and related macro error bullshit that spans 300 lines of errors
|
||||
+//
|
||||
+// Apparently you can't include boost/python stuff _and_ have this header defined
|
||||
+#undef BOOST_MPL_CFG_NO_PREPROCESSED_HEADERS
|
||||
// Boost apparently can't be bothered to fix its own usage of its own
|
||||
// deprecated features.
|
||||
#include <boost/python/extract.hpp>
|
||||
@@ -0,0 +1,403 @@
|
||||
{
|
||||
lib,
|
||||
stdenv,
|
||||
overrideScope,
|
||||
ceph-meta,
|
||||
ceph-src,
|
||||
|
||||
# Build time
|
||||
autoconf,
|
||||
automake,
|
||||
cmake,
|
||||
ensureNewerSourcesHook,
|
||||
fmt,
|
||||
git,
|
||||
libtool,
|
||||
makeWrapper,
|
||||
nasm,
|
||||
pkg-config,
|
||||
which,
|
||||
|
||||
# Tests
|
||||
nixosTests,
|
||||
|
||||
# Runtime dependencies
|
||||
|
||||
arrow-cpp,
|
||||
babeltrace,
|
||||
ceph-boost,
|
||||
bzip2,
|
||||
cryptsetup,
|
||||
cunit,
|
||||
e2fsprogs,
|
||||
doxygen,
|
||||
getopt,
|
||||
gperf,
|
||||
graphviz,
|
||||
gnugrep,
|
||||
gtest,
|
||||
icu,
|
||||
kmod,
|
||||
libcap,
|
||||
libcap_ng,
|
||||
libnbd,
|
||||
libnl,
|
||||
libxml2,
|
||||
lmdb,
|
||||
lttng-ust,
|
||||
# Ceph currently requires >= 5.3
|
||||
lua5_4,
|
||||
lvm2,
|
||||
lz4,
|
||||
oath-toolkit,
|
||||
openldap,
|
||||
parted,
|
||||
ceph-python-env,
|
||||
rdkafka,
|
||||
ceph-rocksdb,
|
||||
snappy,
|
||||
openssh,
|
||||
sqlite,
|
||||
utf8proc,
|
||||
xfsprogs,
|
||||
zlib,
|
||||
zstd,
|
||||
|
||||
# Optional Dependencies
|
||||
curl,
|
||||
expat,
|
||||
fuse,
|
||||
libatomic_ops,
|
||||
libedit,
|
||||
libs3,
|
||||
yasm,
|
||||
|
||||
# Mallocs
|
||||
gperftools,
|
||||
jemalloc,
|
||||
|
||||
# Crypto Dependencies
|
||||
cryptopp,
|
||||
nspr,
|
||||
nss,
|
||||
|
||||
# Linux Only Dependencies
|
||||
linuxHeaders,
|
||||
systemd,
|
||||
util-linux,
|
||||
libuuid,
|
||||
udev,
|
||||
keyutils,
|
||||
rdma-core,
|
||||
rabbitmq-c,
|
||||
libaio,
|
||||
libxfs,
|
||||
liburing,
|
||||
withSystemd ? lib.meta.availableOn stdenv.hostPlatform systemd,
|
||||
}:
|
||||
|
||||
# We must have one crypto library
|
||||
assert cryptopp != null || (nss != null && nspr != null);
|
||||
|
||||
let
|
||||
shouldUsePkg =
|
||||
pkg: if pkg != null && lib.meta.availableOn stdenv.hostPlatform pkg then pkg else null;
|
||||
|
||||
optYasm = shouldUsePkg yasm;
|
||||
optExpat = shouldUsePkg expat;
|
||||
optCurl = shouldUsePkg curl;
|
||||
optFuse = shouldUsePkg fuse;
|
||||
optLibedit = shouldUsePkg libedit;
|
||||
optLibatomic_ops = shouldUsePkg libatomic_ops;
|
||||
optLibs3 = shouldUsePkg libs3;
|
||||
|
||||
optJemalloc = shouldUsePkg jemalloc;
|
||||
optGperftools = shouldUsePkg gperftools;
|
||||
|
||||
optCryptopp = shouldUsePkg cryptopp;
|
||||
optNss = shouldUsePkg nss;
|
||||
optNspr = shouldUsePkg nspr;
|
||||
|
||||
optLibaio = shouldUsePkg libaio;
|
||||
optLibxfs = shouldUsePkg libxfs;
|
||||
|
||||
hasRadosgw = optExpat != null && optCurl != null && optLibedit != null;
|
||||
|
||||
# Malloc implementation (can be jemalloc, tcmalloc or null)
|
||||
malloc = if optJemalloc != null then optJemalloc else optGperftools;
|
||||
|
||||
# We prefer nss over cryptopp
|
||||
cryptoStr =
|
||||
if optNss != null && optNspr != null then
|
||||
"nss"
|
||||
else if optCryptopp != null then
|
||||
"cryptopp"
|
||||
else
|
||||
"none";
|
||||
|
||||
cryptoLibsMap = {
|
||||
nss = [
|
||||
optNss
|
||||
optNspr
|
||||
];
|
||||
cryptopp = [ optCryptopp ];
|
||||
none = [ ];
|
||||
};
|
||||
|
||||
in
|
||||
stdenv.mkDerivation {
|
||||
pname = "ceph";
|
||||
inherit (ceph-src) version;
|
||||
src = ceph-src;
|
||||
|
||||
nativeBuildInputs = [
|
||||
autoconf # `autoreconf` is called, e.g. for `qatlib_ext`
|
||||
automake # `aclocal` is called, e.g. for `qatlib_ext`
|
||||
cmake
|
||||
fmt
|
||||
git
|
||||
makeWrapper
|
||||
libtool # used e.g. for `qatlib_ext`
|
||||
nasm
|
||||
pkg-config
|
||||
ceph-python-env
|
||||
ceph-python-env.pkgs.python # for the toPythonPath function
|
||||
ceph-python-env.pkgs.wrapPython
|
||||
which
|
||||
(ensureNewerSourcesHook { year = "1980"; })
|
||||
# for building docs/man-pages presumably
|
||||
doxygen
|
||||
graphviz
|
||||
];
|
||||
|
||||
buildInputs =
|
||||
cryptoLibsMap.${cryptoStr}
|
||||
++ [
|
||||
arrow-cpp
|
||||
babeltrace
|
||||
ceph-boost
|
||||
bzip2
|
||||
# Adding `ceph-python-env` here adds the env's `site-packages` to `PYTHONPATH` during the build.
|
||||
# This is important, otherwise the build system may not find the Python deps and then
|
||||
# silently skip installing ceph-volume and other Ceph python tools.
|
||||
ceph-python-env
|
||||
cryptsetup
|
||||
cunit
|
||||
e2fsprogs # according to `debian/control` file, `ceph-volume` is supposed to use it
|
||||
gperf
|
||||
gtest
|
||||
icu
|
||||
libcap
|
||||
libnbd
|
||||
libnl
|
||||
libxml2
|
||||
lmdb
|
||||
lttng-ust
|
||||
lua5_4
|
||||
lvm2 # according to `debian/control` file, e.g. `pvs` command used by `src/ceph-volume/ceph_volume/api/lvm.py`
|
||||
lz4
|
||||
malloc
|
||||
oath-toolkit
|
||||
openldap
|
||||
optLibatomic_ops
|
||||
optLibs3
|
||||
optYasm
|
||||
parted # according to `debian/control` file, used by `src/ceph-volume/ceph_volume/util/disk.py`
|
||||
rdkafka
|
||||
ceph-rocksdb
|
||||
snappy
|
||||
openssh # according to `debian/control` file, `ssh` command used by `cephadm`
|
||||
sqlite
|
||||
utf8proc
|
||||
xfsprogs # according to `debian/control` file, `ceph-volume` is supposed to use it
|
||||
zlib
|
||||
zstd
|
||||
]
|
||||
++ lib.optionals stdenv.hostPlatform.isLinux [
|
||||
keyutils
|
||||
libcap_ng
|
||||
liburing
|
||||
libuuid
|
||||
linuxHeaders
|
||||
optLibaio
|
||||
optLibxfs
|
||||
rabbitmq-c
|
||||
rdma-core
|
||||
udev
|
||||
util-linux
|
||||
]
|
||||
++ lib.optionals hasRadosgw [
|
||||
optCurl
|
||||
optExpat
|
||||
optFuse
|
||||
optLibedit
|
||||
];
|
||||
|
||||
inherit (ceph-python-env) sitePackages;
|
||||
|
||||
# Picked up, amongst others, by `wrapPythonPrograms`.
|
||||
pythonPath = [
|
||||
ceph-python-env
|
||||
"${placeholder "out"}/${ceph-python-env.sitePackages}"
|
||||
];
|
||||
|
||||
# * `unset AS` because otherwise the Ceph CMake build errors with
|
||||
# configure: error: No modern nasm or yasm found as required. Nasm should be v2.11.01 or later (v2.13 for AVX512) and yasm should be 1.2.0 or later.
|
||||
# because the code at
|
||||
# https://github.com/intel/isa-l/blob/633add1b569fe927bace3960d7c84ed9c1b38bb9/configure.ac#L99-L191
|
||||
# doesn't even consider using `nasm` or `yasm` but instead uses `$AS`
|
||||
# from `gcc-wrapper`.
|
||||
# (Ceph's error message is extra confusing, because it says
|
||||
# `No modern nasm or yasm found` when in fact it found e.g. `nasm`
|
||||
# but then uses `$AS` instead.
|
||||
# * replace /sbin and /bin based paths with direct nix store paths
|
||||
# * increase the `command` buffer size since 2 nix store paths cannot fit within 128 characters
|
||||
preConfigure = ''
|
||||
unset AS
|
||||
|
||||
substituteInPlace src/common/module.c \
|
||||
--replace-fail "char command[128];" "char command[256];" \
|
||||
--replace-fail "/sbin/modinfo" "${kmod}/bin/modinfo" \
|
||||
--replace-fail "/sbin/modprobe" "${kmod}/bin/modprobe" \
|
||||
--replace-fail "/bin/grep" "${gnugrep}/bin/grep"
|
||||
|
||||
# Patch remount to use full path to mount(8), otherwise ceph-fuse fails when run
|
||||
# from a systemd unit for example.
|
||||
substituteInPlace src/client/fuse_ll.cc \
|
||||
--replace-fail "mount -i -o remount" "${util-linux}/bin/mount -i -o remount"
|
||||
|
||||
substituteInPlace systemd/*.service.in \
|
||||
--replace-quiet "/bin/kill" "${util-linux}/bin/kill"
|
||||
|
||||
substituteInPlace src/{ceph-osd-prestart.sh,ceph-post-file.in,init-ceph.in} \
|
||||
--replace-fail "GETOPT=/usr/local/bin/getopt" "GETOPT=${getopt}/bin/getopt" \
|
||||
--replace-fail "GETOPT=getopt" "GETOPT=${getopt}/bin/getopt"
|
||||
|
||||
# The install target needs to be in PYTHONPATH for "*.pth support" check to succeed
|
||||
export PYTHONPATH=$PYTHONPATH:$lib/$sitePackages:$out/$sitePackages
|
||||
patchShebangs src/
|
||||
'';
|
||||
|
||||
cmakeFlags = [
|
||||
(lib.cmakeOptionType "path" "CMAKE_INSTALL_DATADIR" "${placeholder "lib"}/lib")
|
||||
|
||||
(lib.cmakeBool "WITH_CEPHFS_SHELL" true)
|
||||
(lib.cmakeBool "WITH_SYSTEMD" withSystemd)
|
||||
# Providing a type with this (as is the case with `lib.cmakeOptionType`) triggers an edge case.
|
||||
# This requires an upstream fix/patch as soon as upstream raises the minimum CMake version to 3.21 or higher.
|
||||
# See also: https://github.com/NixOS/nixpkgs/pull/494583#issuecomment-4195176699
|
||||
"-DSYSTEMD_SYSTEM_UNIT_DIR=${placeholder "out"}/lib/systemd/system"
|
||||
# `WITH_JAEGER` requires `thrift` as a depenedncy (fine), but the build fails with:
|
||||
# CMake Error at src/opentelemetry-cpp-stamp/opentelemetry-cpp-build-Release.cmake:49 (message):
|
||||
# Command failed: 2
|
||||
#
|
||||
# 'make' 'opentelemetry_trace' 'opentelemetry_exporter_jaeger_trace'
|
||||
#
|
||||
# See also
|
||||
#
|
||||
# /build/ceph-18.2.0/build/src/opentelemetry-cpp/src/opentelemetry-cpp-stamp/opentelemetry-cpp-build-*.log
|
||||
# and that file contains:
|
||||
# /build/ceph-18.2.0/src/jaegertracing/opentelemetry-cpp/exporters/jaeger/src/TUDPTransport.cc: In member function 'virtual void opentelemetry::v1::exporter::jaeger::TUDPTransport::close()':
|
||||
# /build/ceph-18.2.0/src/jaegertracing/opentelemetry-cpp/exporters/jaeger/src/TUDPTransport.cc:71:7: error: '::close' has not been declared; did you mean 'pclose'?
|
||||
# 71 | ::THRIFT_CLOSESOCKET(socket_);
|
||||
# | ^~~~~~~~~~~~~~~~~~
|
||||
# Looks like `close()` is somehow not included.
|
||||
# But the relevant code is already removed in `open-telemetry` 1.10: https://github.com/open-telemetry/opentelemetry-cpp/pull/2031
|
||||
# So it's probably not worth trying to fix that for this Ceph version,
|
||||
# and instead just disable Ceph's Jaeger support.
|
||||
(lib.cmakeBool "WITH_JAEGER" false)
|
||||
(lib.cmakeBool "WITH_TESTS" false)
|
||||
|
||||
# Use our own libraries, where possible
|
||||
(lib.cmakeBool "WITH_SYSTEM_ARROW" true) # Only used if other options enable Arrow support.
|
||||
(lib.cmakeBool "WITH_SYSTEM_BOOST" true)
|
||||
(lib.cmakeBool "WITH_SYSTEM_GTEST" true)
|
||||
(lib.cmakeBool "WITH_SYSTEM_ROCKSDB" true)
|
||||
(lib.cmakeBool "WITH_SYSTEM_UTF8PROC" true)
|
||||
(lib.cmakeBool "WITH_SYSTEM_ZSTD" true)
|
||||
|
||||
# Use our own python libraries too, see:
|
||||
# https://github.com/NixOS/nixpkgs/pull/344993#issuecomment-2391046329
|
||||
(lib.cmakeFeature "CEPHADM_BUNDLED_DEPENDENCIES" "none")
|
||||
|
||||
# Upstream enables UADK by default on aarch64, but not other platforms.
|
||||
# This causes issues when it tries to fetch the repository on aarch64.
|
||||
# We disable this by default on all platforms, with future PRs adding proper with<Pkg> flags.
|
||||
(lib.cmakeBool "WITH_UADK" false)
|
||||
|
||||
# TODO breaks with sandbox, tries to download stuff with npm
|
||||
(lib.cmakeBool "WITH_MGR_DASHBOARD_FRONTEND" false)
|
||||
# WITH_XFS has been set default ON from Ceph 16, keeping it optional in nixpkgs for now
|
||||
(lib.cmakeBool "WITH_XFS" (optLibxfs != null))
|
||||
]
|
||||
# TODO: investigate setting this to false on other platforms
|
||||
++ lib.optional stdenv.hostPlatform.isLinux (lib.cmakeBool "WITH_SYSTEM_LIBURING" true);
|
||||
|
||||
preBuild =
|
||||
# The legacy-option-headers target is not correctly empbedded in the build graph.
|
||||
# It also contains some internal race conditions that we work around by building with `-j 1`.
|
||||
# Upstream discussion for additional context at https://tracker.ceph.com/issues/63402.
|
||||
''
|
||||
cmake --build . --target legacy-option-headers -j 1
|
||||
'';
|
||||
|
||||
postFixup = ''
|
||||
wrapPythonPrograms
|
||||
# dashboard-related ceph-mgr functionality needs access to the *rados-admin* binary in addition to the python part
|
||||
wrapProgram $out/bin/ceph-mgr \
|
||||
--prefix PYTHONPATH : "$(toPythonPath ${placeholder "out"}):$(toPythonPath ${ceph-python-env})" \
|
||||
--prefix PATH : $out/bin
|
||||
|
||||
# Test that ceph-volume exists since the build system has a tendency to
|
||||
# silently drop it with misconfigurations.
|
||||
test -f $out/bin/ceph-volume
|
||||
|
||||
# Assert that getopt patch from preConfigure covered all instances
|
||||
! grep -F -r 'GETOPT=getopt' $out
|
||||
! grep -F -r 'GETOPT=/usr/local/bin/getopt' $out
|
||||
|
||||
mkdir -p $client/{bin,etc,$sitePackages,share/bash-completion/completions}
|
||||
cp -r $out/bin/{ceph,.ceph-wrapped,rados,rbd,rbdmap} $client/bin
|
||||
cp -r $out/bin/ceph-{authtool,conf,dencoder,rbdnamer,syn} $client/bin
|
||||
cp -r $out/bin/rbd-replay* $client/bin
|
||||
cp -r $out/sbin/mount.ceph $client/bin
|
||||
cp -r $out/sbin/mount.fuse.ceph $client/bin
|
||||
ln -s bin $client/sbin
|
||||
cp -r $out/$sitePackages/* $client/$sitePackages
|
||||
cp -r $out/etc/bash_completion.d $client/share/bash-completion/completions
|
||||
# wrapPythonPrograms modifies .ceph-wrapped, so lets just update its paths
|
||||
substituteInPlace $client/bin/ceph --replace-fail $out $client
|
||||
substituteInPlace $client/bin/.ceph-wrapped --replace-fail $out $client
|
||||
'';
|
||||
|
||||
outputs = [
|
||||
"out"
|
||||
"lib"
|
||||
"client"
|
||||
"dev"
|
||||
"doc"
|
||||
"man"
|
||||
];
|
||||
|
||||
doCheck = false; # uses pip to install things from the internet
|
||||
|
||||
# Takes 7+h to build with 2 cores.
|
||||
requiredSystemFeatures = [ "big-parallel" ];
|
||||
|
||||
meta = ceph-meta "Distributed storage system";
|
||||
|
||||
passthru = {
|
||||
inherit (ceph-src) version;
|
||||
inherit overrideScope;
|
||||
inherit arrow-cpp;
|
||||
pythonEnv = ceph-python-env;
|
||||
tests = {
|
||||
inherit (nixosTests)
|
||||
ceph-multi-node
|
||||
ceph-single-node
|
||||
ceph-single-node-bluestore
|
||||
ceph-single-node-bluestore-dmcrypt
|
||||
;
|
||||
};
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,24 @@
|
||||
{ lib }:
|
||||
description: {
|
||||
homepage = "https://ceph.io/en/";
|
||||
inherit description;
|
||||
license = with lib.licenses; [
|
||||
lgpl21
|
||||
gpl2Only
|
||||
bsd3
|
||||
mit
|
||||
publicDomain
|
||||
];
|
||||
maintainers = with lib.maintainers; [
|
||||
adev
|
||||
ak
|
||||
johanot
|
||||
krav
|
||||
nh2
|
||||
benaryorg
|
||||
];
|
||||
platforms = [
|
||||
"x86_64-linux"
|
||||
"aarch64-linux"
|
||||
];
|
||||
}
|
||||
@@ -1,36 +0,0 @@
|
||||
# This older version only exists because `ceph` needs it, see `cryptography.nix`.
|
||||
{
|
||||
buildPythonPackage,
|
||||
fetchPypi,
|
||||
lib,
|
||||
cryptography,
|
||||
}:
|
||||
|
||||
buildPythonPackage rec {
|
||||
pname = "cryptography-vectors";
|
||||
# The test vectors must have the same version as the cryptography package
|
||||
inherit (cryptography) version;
|
||||
format = "setuptools";
|
||||
|
||||
src = fetchPypi {
|
||||
pname = "cryptography_vectors";
|
||||
inherit version;
|
||||
hash = "sha256-hGBwa1tdDOSoVXHKM4nPiPcAu2oMYTPcn+D1ovW9oEE=";
|
||||
};
|
||||
|
||||
# No tests included
|
||||
doCheck = false;
|
||||
|
||||
pythonImportsCheck = [ "cryptography_vectors" ];
|
||||
|
||||
meta = {
|
||||
description = "Test vectors for the cryptography package";
|
||||
homepage = "https://cryptography.io/en/latest/development/test-vectors/";
|
||||
# Source: https://github.com/pyca/cryptography/tree/master/vectors;
|
||||
license = with lib.licenses; [
|
||||
asl20
|
||||
bsd3
|
||||
];
|
||||
maintainers = with lib.maintainers; [ nh2 ];
|
||||
};
|
||||
}
|
||||
@@ -1,136 +0,0 @@
|
||||
# This older version only exists because `ceph` needs it, see its package.
|
||||
{
|
||||
lib,
|
||||
stdenv,
|
||||
callPackage,
|
||||
buildPythonPackage,
|
||||
fetchPypi,
|
||||
fetchpatch,
|
||||
rustPlatform,
|
||||
cargo,
|
||||
rustc,
|
||||
setuptools-rust,
|
||||
openssl,
|
||||
isPyPy,
|
||||
cffi,
|
||||
pkg-config,
|
||||
pytestCheckHook,
|
||||
pretend,
|
||||
libiconv,
|
||||
iso8601,
|
||||
py,
|
||||
pytz,
|
||||
hypothesis,
|
||||
}:
|
||||
|
||||
let
|
||||
cryptography-vectors = callPackage ./cryptography-vectors.nix { };
|
||||
in
|
||||
buildPythonPackage rec {
|
||||
pname = "cryptography";
|
||||
version = "40.0.1"; # Also update the hash in vectors.nix
|
||||
format = "setuptools";
|
||||
|
||||
src = fetchPypi {
|
||||
inherit pname version;
|
||||
hash = "sha256-KAPy+LHpX2FEGZJsfm9V2CivxhTKXtYVQ4d65mjMNHI=";
|
||||
};
|
||||
|
||||
cargoDeps = rustPlatform.fetchCargoVendor {
|
||||
inherit
|
||||
pname
|
||||
version
|
||||
src
|
||||
cargoRoot
|
||||
;
|
||||
hash = "sha256-pZHu3Oo9DWRAtldU0UvrH1FIg0bEvyfizPUhj9IBL58=";
|
||||
};
|
||||
|
||||
# Since Cryptography v40 is quite outdated, we need to backport
|
||||
# security fixes that are only available in newer versions.
|
||||
patches = [
|
||||
# Fix https://nvd.nist.gov/vuln/detail/CVE-2023-49083 which has no upstream backport.
|
||||
# See https://github.com/pyca/cryptography/commit/f09c261ca10a31fe41b1262306db7f8f1da0e48a#diff-f5134bf8f3cf0a5cc8601df55e50697acc866c603a38caff98802bd8e17976c5R1893
|
||||
./python-cryptography-Cherry-pick-fix-for-CVE-2023-49083-on-cryptography-40.patch
|
||||
|
||||
# Fix https://nvd.nist.gov/vuln/detail/CVE-2024-26130
|
||||
# See https://github.com/pyca/cryptography/commit/97d231672763cdb5959a3b191e692a362f1b9e55
|
||||
(fetchpatch {
|
||||
name = "python-cryptography-CVE-2024-26130-dont-crash-when-a-PKCS-12-key-and-cert-dont-match-mmap-mode.patch";
|
||||
url = "https://github.com/pyca/cryptography/commit/97d231672763cdb5959a3b191e692a362f1b9e55.patch";
|
||||
hash = "sha256-l45NOzOWhHW4nY4OIRpdjYQRvUW8BROGWdpkAtvVn0Y=";
|
||||
})
|
||||
];
|
||||
|
||||
postPatch = ''
|
||||
substituteInPlace pyproject.toml \
|
||||
--replace "--benchmark-disable" ""
|
||||
'';
|
||||
|
||||
cargoRoot = "src/rust";
|
||||
|
||||
build-system = [
|
||||
setuptools-rust
|
||||
];
|
||||
|
||||
nativeBuildInputs = [
|
||||
rustPlatform.cargoSetupHook
|
||||
cargo
|
||||
rustc
|
||||
pkg-config
|
||||
]
|
||||
++ lib.optionals (!isPyPy) [ cffi ];
|
||||
|
||||
buildInputs = [
|
||||
openssl
|
||||
]
|
||||
++ lib.optionals stdenv.hostPlatform.isDarwin [
|
||||
libiconv
|
||||
];
|
||||
|
||||
propagatedBuildInputs = lib.optionals (!isPyPy) [ cffi ];
|
||||
|
||||
nativeCheckInputs = [
|
||||
cryptography-vectors
|
||||
hypothesis
|
||||
iso8601
|
||||
pretend
|
||||
py
|
||||
pytestCheckHook
|
||||
pytz
|
||||
];
|
||||
|
||||
pytestFlags = [ "--disable-pytest-warnings" ];
|
||||
|
||||
disabledTestPaths = [
|
||||
# save compute time by not running benchmarks
|
||||
"tests/bench"
|
||||
# aarch64-darwin forbids W+X memory, but this tests depends on it:
|
||||
# * https://cffi.readthedocs.io/en/latest/using.html#callbacks
|
||||
# furthermore, this test fails with OpenSSL 3.6.0, probably due to:
|
||||
# * https://github.com/openssl/openssl/issues/28757
|
||||
# * https://github.com/openssl/openssl/issues/28770
|
||||
# * https://github.com/openssl/openssl/issues/28888
|
||||
"tests/hazmat/backends/test_openssl_memleak.py"
|
||||
];
|
||||
|
||||
meta = {
|
||||
description = "Package which provides cryptographic recipes and primitives";
|
||||
longDescription = ''
|
||||
Cryptography includes both high level recipes and low level interfaces to
|
||||
common cryptographic algorithms such as symmetric ciphers, message
|
||||
digests, and key derivation functions.
|
||||
Our goal is for it to be your "cryptographic standard library". It
|
||||
supports Python 2.7, Python 3.5+, and PyPy 5.4+.
|
||||
'';
|
||||
homepage = "https://github.com/pyca/cryptography";
|
||||
changelog =
|
||||
"https://cryptography.io/en/latest/changelog/#v" + lib.replaceStrings [ "." ] [ "-" ] version;
|
||||
license = with lib.licenses; [
|
||||
asl20
|
||||
bsd3
|
||||
psfl
|
||||
];
|
||||
maintainers = with lib.maintainers; [ nh2 ];
|
||||
};
|
||||
}
|
||||
-44
@@ -1,44 +0,0 @@
|
||||
commit 08dbad8552ecca85d3e38072f94eb864b143d218
|
||||
Author: Niklas Hambüchen <mail@nh2.me>
|
||||
Date: Mon Jan 22 12:52:06 2024 +0000
|
||||
|
||||
Cherry-pick fix for CVE-2023-49083
|
||||
|
||||
Cherry-Picked-From: f09c261ca10a31fe41b1262306db7f8f1da0e48a
|
||||
Cherry-Picked-By: Niklas Hambüchen <mail@nh2.me>
|
||||
|
||||
diff --git a/src/cryptography/hazmat/backends/openssl/backend.py b/src/cryptography/hazmat/backends/openssl/backend.py
|
||||
index a3fe1bce4..58e7207c4 100644
|
||||
--- a/src/cryptography/hazmat/backends/openssl/backend.py
|
||||
+++ b/src/cryptography/hazmat/backends/openssl/backend.py
|
||||
@@ -2383,9 +2383,12 @@ class Backend:
|
||||
_Reasons.UNSUPPORTED_SERIALIZATION,
|
||||
)
|
||||
|
||||
+ certs: list[x509.Certificate] = []
|
||||
+ if p7.d.sign == self._ffi.NULL:
|
||||
+ return certs
|
||||
+
|
||||
sk_x509 = p7.d.sign.cert
|
||||
num = self._lib.sk_X509_num(sk_x509)
|
||||
- certs = []
|
||||
for i in range(num):
|
||||
x509 = self._lib.sk_X509_value(sk_x509, i)
|
||||
self.openssl_assert(x509 != self._ffi.NULL)
|
||||
diff --git a/tests/hazmat/primitives/test_pkcs7.py b/tests/hazmat/primitives/test_pkcs7.py
|
||||
index 4e61c5ef5..d8170bfb9 100644
|
||||
--- a/tests/hazmat/primitives/test_pkcs7.py
|
||||
+++ b/tests/hazmat/primitives/test_pkcs7.py
|
||||
@@ -89,6 +89,12 @@ class TestPKCS7Loading:
|
||||
mode="rb",
|
||||
)
|
||||
|
||||
+ def test_load_pkcs7_empty_certificates(self, backend):
|
||||
+ der = b"\x30\x0B\x06\x09\x2A\x86\x48\x86\xF7\x0D\x01\x07\x02"
|
||||
+
|
||||
+ certificates = pkcs7.load_der_pkcs7_certificates(der)
|
||||
+ assert certificates == []
|
||||
+
|
||||
|
||||
# We have no public verification API and won't be adding one until we get
|
||||
# some requirements from users so this function exists to give us basic
|
||||
@@ -1,49 +0,0 @@
|
||||
# This older version only exists because `ceph` needs it, see its package.
|
||||
{
|
||||
lib,
|
||||
buildPythonPackage,
|
||||
cryptography,
|
||||
fetchPypi,
|
||||
idna,
|
||||
pyopenssl,
|
||||
pytestCheckHook,
|
||||
service-identity,
|
||||
}:
|
||||
|
||||
buildPythonPackage rec {
|
||||
pname = "trustme";
|
||||
version = "1.1.0";
|
||||
format = "setuptools";
|
||||
|
||||
src = fetchPypi {
|
||||
inherit pname version;
|
||||
hash = "sha256-U3Wtf7QnB0vslWWS4NTuKkz02miTThukvPQhcSa8ReY=";
|
||||
};
|
||||
|
||||
propagatedBuildInputs = [
|
||||
cryptography
|
||||
idna
|
||||
];
|
||||
|
||||
nativeCheckInputs = [
|
||||
pyopenssl
|
||||
pytestCheckHook
|
||||
service-identity
|
||||
];
|
||||
|
||||
# Some of the tests use localhost networking.
|
||||
__darwinAllowLocalNetworking = true;
|
||||
|
||||
pythonImportsCheck = [ "trustme" ];
|
||||
|
||||
meta = {
|
||||
description = "High quality TLS certs while you wait, for the discerning tester";
|
||||
homepage = "https://github.com/python-trio/trustme";
|
||||
changelog = "https://trustme.readthedocs.io/en/latest/#change-history";
|
||||
license = with lib.licenses; [
|
||||
mit
|
||||
asl20
|
||||
];
|
||||
maintainers = with lib.maintainers; [ jfly ];
|
||||
};
|
||||
}
|
||||
@@ -1,664 +1,6 @@
|
||||
{
|
||||
lib,
|
||||
stdenv,
|
||||
runCommand,
|
||||
fetchurl,
|
||||
fetchFromGitHub,
|
||||
fetchPypi,
|
||||
fetchpatch2,
|
||||
callPackage,
|
||||
|
||||
# Build time
|
||||
autoconf,
|
||||
automake,
|
||||
cmake,
|
||||
ensureNewerSourcesHook,
|
||||
# To see which `fmt` version Ceph upstream recommends, check its `src/fmt` submodule.
|
||||
#
|
||||
# Ceph does not currently build with `fmt_10`; see https://github.com/NixOS/nixpkgs/issues/281027#issuecomment-1899128557
|
||||
# If we want to switch for that before upstream fixes it, use this patch:
|
||||
# https://github.com/NixOS/nixpkgs/pull/281858#issuecomment-1899648638
|
||||
fmt_9,
|
||||
git,
|
||||
libtool,
|
||||
makeWrapper,
|
||||
nasm,
|
||||
pkg-config,
|
||||
which,
|
||||
openssl,
|
||||
|
||||
# Tests
|
||||
nixosTests,
|
||||
|
||||
# Runtime dependencies
|
||||
|
||||
# Remove once Ceph supports arrow-cpp >= 20, see:
|
||||
# * https://tracker.ceph.com/issues/71269
|
||||
# * https://github.com/NixOS/nixpkgs/issues/406306
|
||||
ceph-arrow-cpp ? callPackage ./arrow-cpp-19.nix { },
|
||||
babeltrace,
|
||||
# Note when trying to upgrade boost:
|
||||
# * When upgrading Ceph, it's recommended to check which boost version Ceph uses on Fedora,
|
||||
# and default to that.
|
||||
# * The version that Ceph downloads if `-DWITH_SYSTEM_BOOST:BOOL=ON` is not given
|
||||
# is declared in `cmake/modules/BuildBoost.cmake` line `set(boost_version ...)`.
|
||||
#
|
||||
# If you want to upgrade to boost >= 1.86, you need a Ceph version that
|
||||
# has this PR in:
|
||||
# https://github.com/ceph/ceph/pull/61312
|
||||
boost183,
|
||||
bzip2,
|
||||
cryptsetup,
|
||||
cunit,
|
||||
e2fsprogs,
|
||||
doxygen,
|
||||
getopt,
|
||||
gperf,
|
||||
graphviz,
|
||||
gnugrep,
|
||||
gtest,
|
||||
icu,
|
||||
kmod,
|
||||
libcap,
|
||||
libcap_ng,
|
||||
libnbd,
|
||||
libnl,
|
||||
libxml2,
|
||||
lmdb,
|
||||
lttng-ust,
|
||||
# Ceph currently requires >= 5.3
|
||||
lua5_4,
|
||||
lvm2,
|
||||
lz4,
|
||||
oath-toolkit,
|
||||
openldap,
|
||||
parted,
|
||||
python311, # to get an idea which Python versions are supported by Ceph, see upstream `do_cmake.sh` (see `PYBUILD=` variable)
|
||||
rdkafka,
|
||||
rocksdb,
|
||||
snappy,
|
||||
openssh,
|
||||
sqlite,
|
||||
utf8proc,
|
||||
xfsprogs,
|
||||
zlib,
|
||||
zstd,
|
||||
|
||||
# Dependencies of overridden Python dependencies, hopefully we can remove these soon.
|
||||
rustPlatform,
|
||||
|
||||
# Optional Dependencies
|
||||
curl ? null,
|
||||
expat ? null,
|
||||
fuse ? null,
|
||||
libatomic_ops ? null,
|
||||
libedit ? null,
|
||||
libs3 ? null,
|
||||
yasm ? null,
|
||||
|
||||
# Mallocs
|
||||
gperftools ? null,
|
||||
jemalloc ? null,
|
||||
|
||||
# Crypto Dependencies
|
||||
cryptopp ? null,
|
||||
nspr ? null,
|
||||
nss ? null,
|
||||
|
||||
# Linux Only Dependencies
|
||||
linuxHeaders,
|
||||
systemd,
|
||||
util-linux,
|
||||
libuuid,
|
||||
udev,
|
||||
keyutils,
|
||||
rdma-core,
|
||||
rabbitmq-c,
|
||||
libaio ? null,
|
||||
libxfs ? null,
|
||||
liburing ? null,
|
||||
zfs ? null,
|
||||
withSystemd ? lib.meta.availableOn stdenv.hostPlatform systemd,
|
||||
}:
|
||||
|
||||
# We must have one crypto library
|
||||
assert cryptopp != null || (nss != null && nspr != null);
|
||||
{ callPackage }:
|
||||
|
||||
let
|
||||
shouldUsePkg =
|
||||
pkg: if pkg != null && lib.meta.availableOn stdenv.hostPlatform pkg then pkg else null;
|
||||
|
||||
optYasm = shouldUsePkg yasm;
|
||||
optExpat = shouldUsePkg expat;
|
||||
optCurl = shouldUsePkg curl;
|
||||
optFuse = shouldUsePkg fuse;
|
||||
optLibedit = shouldUsePkg libedit;
|
||||
optLibatomic_ops = shouldUsePkg libatomic_ops;
|
||||
optLibs3 = shouldUsePkg libs3;
|
||||
|
||||
optJemalloc = shouldUsePkg jemalloc;
|
||||
optGperftools = shouldUsePkg gperftools;
|
||||
|
||||
optCryptopp = shouldUsePkg cryptopp;
|
||||
optNss = shouldUsePkg nss;
|
||||
optNspr = shouldUsePkg nspr;
|
||||
|
||||
optLibaio = shouldUsePkg libaio;
|
||||
optLibxfs = shouldUsePkg libxfs;
|
||||
optZfs = shouldUsePkg zfs;
|
||||
|
||||
# Downgrade rocksdb, 7.10 breaks ceph
|
||||
rocksdb' = rocksdb.overrideAttrs {
|
||||
version = "7.9.2";
|
||||
src = fetchFromGitHub {
|
||||
owner = "facebook";
|
||||
repo = "rocksdb";
|
||||
tag = "v7.9.2";
|
||||
hash = "sha256-5P7IqJ14EZzDkbjaBvbix04ceGGdlWBuVFH/5dpD5VM=";
|
||||
};
|
||||
};
|
||||
|
||||
hasRadosgw = optExpat != null && optCurl != null && optLibedit != null;
|
||||
|
||||
# Malloc implementation (can be jemalloc, tcmalloc or null)
|
||||
malloc = if optJemalloc != null then optJemalloc else optGperftools;
|
||||
|
||||
# We prefer nss over cryptopp
|
||||
cryptoStr =
|
||||
if optNss != null && optNspr != null then
|
||||
"nss"
|
||||
else if optCryptopp != null then
|
||||
"cryptopp"
|
||||
else
|
||||
"none";
|
||||
|
||||
cryptoLibsMap = {
|
||||
nss = [
|
||||
optNss
|
||||
optNspr
|
||||
];
|
||||
cryptopp = [ optCryptopp ];
|
||||
none = [ ];
|
||||
};
|
||||
|
||||
getMeta = description: {
|
||||
homepage = "https://ceph.io/en/";
|
||||
inherit description;
|
||||
license = with lib.licenses; [
|
||||
lgpl21
|
||||
gpl2Only
|
||||
bsd3
|
||||
mit
|
||||
publicDomain
|
||||
];
|
||||
maintainers = with lib.maintainers; [
|
||||
adev
|
||||
ak
|
||||
johanot
|
||||
krav
|
||||
nh2
|
||||
benaryorg
|
||||
];
|
||||
platforms = [
|
||||
"x86_64-linux"
|
||||
"aarch64-linux"
|
||||
];
|
||||
};
|
||||
|
||||
ceph-common =
|
||||
with python.pkgs;
|
||||
buildPythonPackage {
|
||||
pname = "ceph-common";
|
||||
format = "setuptools";
|
||||
inherit src version;
|
||||
|
||||
sourceRoot = "ceph-${version}/src/python-common";
|
||||
|
||||
propagatedBuildInputs = [
|
||||
pyyaml
|
||||
];
|
||||
|
||||
nativeCheckInputs = [
|
||||
pytestCheckHook
|
||||
];
|
||||
|
||||
disabledTests = [
|
||||
# requires network access
|
||||
"test_valid_addr"
|
||||
];
|
||||
|
||||
meta = getMeta "Ceph common module for code shared by manager modules";
|
||||
};
|
||||
|
||||
# Watch out for python <> boost compatibility
|
||||
python = python311.override {
|
||||
self = python;
|
||||
packageOverrides =
|
||||
self: super:
|
||||
let
|
||||
bcryptOverrideVersion = "4.0.1";
|
||||
in
|
||||
{
|
||||
# Ceph does not support the following yet:
|
||||
# * `bcrypt` > 4.0
|
||||
# * `cryptography` > 40
|
||||
# See:
|
||||
# * https://github.com/NixOS/nixpkgs/pull/281858#issuecomment-1899358602
|
||||
# * Upstream issue: https://tracker.ceph.com/issues/63529
|
||||
# > Python Sub-Interpreter Model Used by ceph-mgr Incompatible With Python Modules Based on PyO3
|
||||
# * Moved to issue: https://tracker.ceph.com/issues/64213
|
||||
# > MGR modules incompatible with later PyO3 versions - PyO3 modules may only be initialized once per interpreter process
|
||||
|
||||
bcrypt = super.bcrypt.overridePythonAttrs (old: rec {
|
||||
pname = "bcrypt";
|
||||
version = bcryptOverrideVersion;
|
||||
src = fetchPypi {
|
||||
inherit pname version;
|
||||
hash = "sha256-J9N1kDrIJhz+QEf2cJ0W99GNObHskqr3KvmJVSplDr0=";
|
||||
};
|
||||
cargoRoot = "src/_bcrypt";
|
||||
cargoDeps = rustPlatform.fetchCargoVendor {
|
||||
inherit
|
||||
pname
|
||||
version
|
||||
src
|
||||
cargoRoot
|
||||
;
|
||||
hash = "sha256-8PyCgh/rUO8uynzGdgylAsb5k55dP9fCnf40UOTCR/M=";
|
||||
};
|
||||
});
|
||||
|
||||
# We pin the older `cryptography` 40 here;
|
||||
# this also forces us to pin other packages, see below
|
||||
cryptography = self.callPackage ./old-python-packages/cryptography.nix { };
|
||||
|
||||
# This is the most recent version of `pyopenssl` that's still compatible with `cryptography` 40.
|
||||
# See https://github.com/NixOS/nixpkgs/pull/281858#issuecomment-1899358602
|
||||
# and https://github.com/pyca/pyopenssl/blob/d9752e44127ba36041b045417af8a0bf16ec4f1e/CHANGELOG.rst#2320-2023-05-30
|
||||
pyopenssl = super.pyopenssl.overridePythonAttrs (old: rec {
|
||||
version = "23.1.1";
|
||||
src = fetchPypi {
|
||||
pname = "pyOpenSSL";
|
||||
inherit version;
|
||||
hash = "sha256-hBSYub7GFiOxtsR+u8AjZ8B9YODhlfGXkIF/EMyNsLc=";
|
||||
};
|
||||
disabledTests = old.disabledTests or [ ] ++ [
|
||||
"test_export_md5_digest"
|
||||
];
|
||||
disabledTestPaths = old.disabledTestPaths or [ ] ++ [
|
||||
"tests/test_ssl.py"
|
||||
];
|
||||
propagatedBuildInputs = old.propagatedBuildInputs or [ ] ++ [
|
||||
self.flaky
|
||||
];
|
||||
# hack: avoid building docs due to incompatibility with current sphinx
|
||||
nativeBuildInputs = [ openssl ]; # old.nativeBuildInputs but without sphinx*
|
||||
outputs = lib.filter (o: o != "doc") old.outputs;
|
||||
});
|
||||
|
||||
# This is the most recent version of `trustme` that's still compatible with `cryptography` 40.
|
||||
# See https://github.com/NixOS/nixpkgs/issues/359723
|
||||
# and https://github.com/python-trio/trustme/commit/586f7759d5c27beb44da60615a71848eb2a5a490
|
||||
trustme = self.callPackage ./old-python-packages/trustme.nix { };
|
||||
|
||||
fastapi = super.fastapi.overridePythonAttrs (old: {
|
||||
# Flaky test:
|
||||
# ResourceWarning: Unclosed <MemoryObjectSendStream>
|
||||
# Unclear whether it's flaky in general or only in this overridden package set.
|
||||
doCheck = false;
|
||||
});
|
||||
|
||||
# Ceph does not support `kubernetes` >= 19, see:
|
||||
# https://github.com/NixOS/nixpkgs/pull/281858#issuecomment-1900324090
|
||||
kubernetes = super.kubernetes.overridePythonAttrs (old: rec {
|
||||
version = "18.20.0";
|
||||
src = fetchFromGitHub {
|
||||
owner = "kubernetes-client";
|
||||
repo = "python";
|
||||
rev = "v${version}";
|
||||
sha256 = "1sawp62j7h0yksmg9jlv4ik9b9i1a1w9syywc9mv8x89wibf5ql1";
|
||||
fetchSubmodules = true;
|
||||
};
|
||||
});
|
||||
|
||||
};
|
||||
};
|
||||
|
||||
boost' = boost183.override {
|
||||
enablePython = true;
|
||||
inherit python;
|
||||
};
|
||||
|
||||
# TODO: split this off in build and runtime environment
|
||||
ceph-python-env = python.withPackages (
|
||||
ps: with ps; [
|
||||
ceph-common
|
||||
|
||||
# build time
|
||||
cython_0
|
||||
|
||||
# debian/control
|
||||
bcrypt
|
||||
cherrypy
|
||||
influxdb
|
||||
jinja2
|
||||
kubernetes
|
||||
natsort
|
||||
numpy
|
||||
pecan
|
||||
prettytable
|
||||
pyjwt
|
||||
pyopenssl
|
||||
python-dateutil
|
||||
pyyaml
|
||||
requests
|
||||
routes
|
||||
scikit-learn
|
||||
scipy
|
||||
setuptools
|
||||
sphinx
|
||||
virtualenv
|
||||
werkzeug
|
||||
|
||||
# src/cephadm/zipapp-reqs.txt
|
||||
markupsafe
|
||||
|
||||
# src/pybind/mgr/requirements-required.txt
|
||||
cryptography
|
||||
jsonpatch
|
||||
|
||||
# src/tools/cephfs/shell/setup.py
|
||||
cmd2
|
||||
colorama
|
||||
]
|
||||
);
|
||||
inherit (ceph-python-env.python) sitePackages;
|
||||
|
||||
version = "19.2.3";
|
||||
src = fetchurl {
|
||||
url = "https://download.ceph.com/tarballs/ceph-${version}.tar.gz";
|
||||
hash = "sha256-zlgp28C81SZbaFJ4yvQk4ZgYz4K/aZqtcISTO8LscSU=";
|
||||
};
|
||||
scope = callPackage ./scope.nix { };
|
||||
in
|
||||
stdenv.mkDerivation {
|
||||
pname = "ceph";
|
||||
inherit src version;
|
||||
|
||||
patches = [
|
||||
./boost-1.85.patch
|
||||
|
||||
(fetchpatch2 {
|
||||
name = "ceph-boost-1.86-uuid.patch";
|
||||
url = "https://github.com/ceph/ceph/commit/01306208eac492ee0e67bff143fc32d0551a2a6f.patch?full_index=1";
|
||||
hash = "sha256-OnDrr72inzGXXYxPFQevsRZImSvI0uuqFHqtFU2dPQE=";
|
||||
})
|
||||
|
||||
# See:
|
||||
# * <https://github.com/boostorg/python/issues/394>
|
||||
# * <https://aur.archlinux.org/cgit/aur.git/commit/?h=ceph&id=8c5cc7d8deec002f7596b6d0860859a0a718f12b>
|
||||
# * <https://github.com/ceph/ceph/pull/60999>
|
||||
./boost-1.86-PyModule.patch
|
||||
|
||||
(fetchpatch2 {
|
||||
name = "ceph-cmake-4.patch";
|
||||
url = "https://gitlab.alpinelinux.org/ashpool/aports/-/raw/d22b70eafe33c3daabe4eea6913c5be87d9463ad/community/ceph19/cpp_redis.patch";
|
||||
hash = "sha256-wxPIsYt25CjXhJ6kmr/MXwFD58Sl4y4W+r9jAMND+uw=";
|
||||
})
|
||||
|
||||
# See:
|
||||
# * <https://github.com/ceph/ceph/pull/55560>
|
||||
# * <https://github.com/ceph/ceph/pull/60575>
|
||||
(fetchpatch2 {
|
||||
name = "ceph-systemd-sans-cluster-name.patch";
|
||||
url = "https://github.com/ceph/ceph/commit/5659920c7c128cb8d9552580dbe23dd167a56c31.patch?full_index=1";
|
||||
hash = "sha256-Uch8ZghyTowUvSq0p/RxiVpdG1Yqlww9inpVksO6zyk=";
|
||||
})
|
||||
(fetchpatch2 {
|
||||
name = "ceph-systemd-prefix.patch";
|
||||
url = "https://github.com/ceph/ceph/commit/9b38df488d7101b02afa834ea518fd52076d582a.patch?full_index=1";
|
||||
hash = "sha256-VcbJhCGTUdNISBd6P96Mm5M3fFVmZ8r7pMl+srQmnIQ=";
|
||||
})
|
||||
(fetchpatch2 {
|
||||
name = "ceph-19.2.2-gcc15.patch";
|
||||
url = "https://github.com/ceph/ceph/commit/830925f0dd196f920893b1947ae74171a202e825.patch";
|
||||
hash = "sha256-bs+noyjiyAjwqfgSHDxdZJnZ/kptOOcz75KMqAaROpg=";
|
||||
})
|
||||
];
|
||||
|
||||
nativeBuildInputs = [
|
||||
autoconf # `autoreconf` is called, e.g. for `qatlib_ext`
|
||||
automake # `aclocal` is called, e.g. for `qatlib_ext`
|
||||
cmake
|
||||
fmt_9
|
||||
git
|
||||
makeWrapper
|
||||
libtool # used e.g. for `qatlib_ext`
|
||||
nasm
|
||||
pkg-config
|
||||
python
|
||||
python.pkgs.python # for the toPythonPath function
|
||||
python.pkgs.wrapPython
|
||||
which
|
||||
(ensureNewerSourcesHook { year = "1980"; })
|
||||
# for building docs/man-pages presumably
|
||||
doxygen
|
||||
graphviz
|
||||
];
|
||||
|
||||
buildInputs =
|
||||
cryptoLibsMap.${cryptoStr}
|
||||
++ [
|
||||
ceph-arrow-cpp
|
||||
babeltrace
|
||||
boost'
|
||||
bzip2
|
||||
# Adding `ceph-python-env` here adds the env's `site-packages` to `PYTHONPATH` during the build.
|
||||
# This is important, otherwise the build system may not find the Python deps and then
|
||||
# silently skip installing ceph-volume and other Ceph python tools.
|
||||
ceph-python-env
|
||||
cryptsetup
|
||||
cunit
|
||||
e2fsprogs # according to `debian/control` file, `ceph-volume` is supposed to use it
|
||||
gperf
|
||||
gtest
|
||||
icu
|
||||
libcap
|
||||
libnbd
|
||||
libnl
|
||||
libxml2
|
||||
lmdb
|
||||
lttng-ust
|
||||
lua5_4
|
||||
lvm2 # according to `debian/control` file, e.g. `pvs` command used by `src/ceph-volume/ceph_volume/api/lvm.py`
|
||||
lz4
|
||||
malloc
|
||||
oath-toolkit
|
||||
openldap
|
||||
optLibatomic_ops
|
||||
optLibs3
|
||||
optYasm
|
||||
parted # according to `debian/control` file, used by `src/ceph-volume/ceph_volume/util/disk.py`
|
||||
rdkafka
|
||||
rocksdb'
|
||||
snappy
|
||||
openssh # according to `debian/control` file, `ssh` command used by `cephadm`
|
||||
sqlite
|
||||
utf8proc
|
||||
xfsprogs # according to `debian/control` file, `ceph-volume` is supposed to use it
|
||||
zlib
|
||||
zstd
|
||||
]
|
||||
++ lib.optionals stdenv.hostPlatform.isLinux [
|
||||
keyutils
|
||||
libcap_ng
|
||||
liburing
|
||||
libuuid
|
||||
linuxHeaders
|
||||
optLibaio
|
||||
optLibxfs
|
||||
optZfs
|
||||
rabbitmq-c
|
||||
rdma-core
|
||||
udev
|
||||
util-linux
|
||||
]
|
||||
++ lib.optionals hasRadosgw [
|
||||
optCurl
|
||||
optExpat
|
||||
optFuse
|
||||
optLibedit
|
||||
];
|
||||
|
||||
# Picked up, amongst others, by `wrapPythonPrograms`.
|
||||
pythonPath = [
|
||||
ceph-python-env
|
||||
"${placeholder "out"}/${ceph-python-env.sitePackages}"
|
||||
];
|
||||
|
||||
# * `unset AS` because otherwise the Ceph CMake build errors with
|
||||
# configure: error: No modern nasm or yasm found as required. Nasm should be v2.11.01 or later (v2.13 for AVX512) and yasm should be 1.2.0 or later.
|
||||
# because the code at
|
||||
# https://github.com/intel/isa-l/blob/633add1b569fe927bace3960d7c84ed9c1b38bb9/configure.ac#L99-L191
|
||||
# doesn't even consider using `nasm` or `yasm` but instead uses `$AS`
|
||||
# from `gcc-wrapper`.
|
||||
# (Ceph's error message is extra confusing, because it says
|
||||
# `No modern nasm or yasm found` when in fact it found e.g. `nasm`
|
||||
# but then uses `$AS` instead.
|
||||
# * replace /sbin and /bin based paths with direct nix store paths
|
||||
# * increase the `command` buffer size since 2 nix store paths cannot fit within 128 characters
|
||||
preConfigure = ''
|
||||
unset AS
|
||||
|
||||
substituteInPlace src/common/module.c \
|
||||
--replace "char command[128];" "char command[256];" \
|
||||
--replace "/sbin/modinfo" "${kmod}/bin/modinfo" \
|
||||
--replace "/sbin/modprobe" "${kmod}/bin/modprobe" \
|
||||
--replace "/bin/grep" "${gnugrep}/bin/grep"
|
||||
|
||||
# Patch remount to use full path to mount(8), otherwise ceph-fuse fails when run
|
||||
# from a systemd unit for example.
|
||||
substituteInPlace src/client/fuse_ll.cc \
|
||||
--replace-fail "mount -i -o remount" "${util-linux}/bin/mount -i -o remount"
|
||||
|
||||
substituteInPlace systemd/*.service.in \
|
||||
--replace-quiet "/bin/kill" "${util-linux}/bin/kill"
|
||||
|
||||
substituteInPlace src/{ceph-osd-prestart.sh,ceph-post-file.in,init-ceph.in} \
|
||||
--replace-fail "GETOPT=/usr/local/bin/getopt" "GETOPT=${getopt}/bin/getopt" \
|
||||
--replace-fail "GETOPT=getopt" "GETOPT=${getopt}/bin/getopt"
|
||||
|
||||
# The install target needs to be in PYTHONPATH for "*.pth support" check to succeed
|
||||
export PYTHONPATH=$PYTHONPATH:$lib/${sitePackages}:$out/${sitePackages}
|
||||
patchShebangs src/
|
||||
'';
|
||||
|
||||
cmakeFlags = [
|
||||
"-DCMAKE_INSTALL_DATADIR=${placeholder "lib"}/lib"
|
||||
|
||||
"-DWITH_CEPHFS_SHELL:BOOL=ON"
|
||||
"-DWITH_SYSTEMD:BOOL=${if withSystemd then "ON" else "OFF"}"
|
||||
"-DSYSTEMD_SYSTEM_UNIT_DIR=${placeholder "out"}/lib/systemd/system"
|
||||
# `WITH_JAEGER` requires `thrift` as a depenedncy (fine), but the build fails with:
|
||||
# CMake Error at src/opentelemetry-cpp-stamp/opentelemetry-cpp-build-Release.cmake:49 (message):
|
||||
# Command failed: 2
|
||||
#
|
||||
# 'make' 'opentelemetry_trace' 'opentelemetry_exporter_jaeger_trace'
|
||||
#
|
||||
# See also
|
||||
#
|
||||
# /build/ceph-18.2.0/build/src/opentelemetry-cpp/src/opentelemetry-cpp-stamp/opentelemetry-cpp-build-*.log
|
||||
# and that file contains:
|
||||
# /build/ceph-18.2.0/src/jaegertracing/opentelemetry-cpp/exporters/jaeger/src/TUDPTransport.cc: In member function 'virtual void opentelemetry::v1::exporter::jaeger::TUDPTransport::close()':
|
||||
# /build/ceph-18.2.0/src/jaegertracing/opentelemetry-cpp/exporters/jaeger/src/TUDPTransport.cc:71:7: error: '::close' has not been declared; did you mean 'pclose'?
|
||||
# 71 | ::THRIFT_CLOSESOCKET(socket_);
|
||||
# | ^~~~~~~~~~~~~~~~~~
|
||||
# Looks like `close()` is somehow not included.
|
||||
# But the relevant code is already removed in `open-telemetry` 1.10: https://github.com/open-telemetry/opentelemetry-cpp/pull/2031
|
||||
# So it's probably not worth trying to fix that for this Ceph version,
|
||||
# and instead just disable Ceph's Jaeger support.
|
||||
"-DWITH_JAEGER:BOOL=OFF"
|
||||
"-DWITH_TESTS:BOOL=OFF"
|
||||
|
||||
# Use our own libraries, where possible
|
||||
"-DWITH_SYSTEM_ARROW:BOOL=ON" # Only used if other options enable Arrow support.
|
||||
"-DWITH_SYSTEM_BOOST:BOOL=ON"
|
||||
"-DWITH_SYSTEM_GTEST:BOOL=ON"
|
||||
"-DWITH_SYSTEM_ROCKSDB:BOOL=ON"
|
||||
"-DWITH_SYSTEM_UTF8PROC:BOOL=ON"
|
||||
"-DWITH_SYSTEM_ZSTD:BOOL=ON"
|
||||
|
||||
# Use our own python libraries too, see:
|
||||
# https://github.com/NixOS/nixpkgs/pull/344993#issuecomment-2391046329
|
||||
"-DCEPHADM_BUNDLED_DEPENDENCIES=none"
|
||||
|
||||
# TODO breaks with sandbox, tries to download stuff with npm
|
||||
"-DWITH_MGR_DASHBOARD_FRONTEND:BOOL=OFF"
|
||||
# WITH_XFS has been set default ON from Ceph 16, keeping it optional in nixpkgs for now
|
||||
"-DWITH_XFS=${if optLibxfs != null then "ON" else "OFF"}"
|
||||
]
|
||||
++ lib.optional stdenv.hostPlatform.isLinux "-DWITH_SYSTEM_LIBURING=ON";
|
||||
|
||||
preBuild =
|
||||
# The legacy-option-headers target is not correctly empbedded in the build graph.
|
||||
# It also contains some internal race conditions that we work around by building with `-j 1`.
|
||||
# Upstream discussion for additional context at https://tracker.ceph.com/issues/63402.
|
||||
''
|
||||
cmake --build . --target legacy-option-headers -j 1
|
||||
'';
|
||||
|
||||
postFixup = ''
|
||||
wrapPythonPrograms
|
||||
wrapProgram $out/bin/ceph-mgr --prefix PYTHONPATH ":" "$(toPythonPath ${placeholder "out"}):$(toPythonPath ${ceph-python-env})"
|
||||
|
||||
# Test that ceph-volume exists since the build system has a tendency to
|
||||
# silently drop it with misconfigurations.
|
||||
test -f $out/bin/ceph-volume
|
||||
|
||||
# Assert that getopt patch from preConfigure covered all instances
|
||||
! grep -F -r 'GETOPT=getopt' $out
|
||||
! grep -F -r 'GETOPT=/usr/local/bin/getopt' $out
|
||||
|
||||
mkdir -p $client/{bin,etc,${sitePackages},share/bash-completion/completions}
|
||||
cp -r $out/bin/{ceph,.ceph-wrapped,rados,rbd,rbdmap} $client/bin
|
||||
cp -r $out/bin/ceph-{authtool,conf,dencoder,rbdnamer,syn} $client/bin
|
||||
cp -r $out/bin/rbd-replay* $client/bin
|
||||
cp -r $out/sbin/mount.ceph $client/bin
|
||||
cp -r $out/sbin/mount.fuse.ceph $client/bin
|
||||
ln -s bin $client/sbin
|
||||
cp -r $out/${sitePackages}/* $client/${sitePackages}
|
||||
cp -r $out/etc/bash_completion.d $client/share/bash-completion/completions
|
||||
# wrapPythonPrograms modifies .ceph-wrapped, so lets just update its paths
|
||||
substituteInPlace $client/bin/ceph --replace $out $client
|
||||
substituteInPlace $client/bin/.ceph-wrapped --replace $out $client
|
||||
'';
|
||||
|
||||
outputs = [
|
||||
"out"
|
||||
"lib"
|
||||
"client"
|
||||
"dev"
|
||||
"doc"
|
||||
"man"
|
||||
];
|
||||
|
||||
doCheck = false; # uses pip to install things from the internet
|
||||
|
||||
# Takes 7+h to build with 2 cores.
|
||||
requiredSystemFeatures = [ "big-parallel" ];
|
||||
|
||||
meta = getMeta "Distributed storage system";
|
||||
|
||||
passthru = {
|
||||
inherit version;
|
||||
inherit python; # to be able to test our overridden packages above individually with `nix-build -A`
|
||||
arrow-cpp = ceph-arrow-cpp;
|
||||
tests = {
|
||||
inherit (nixosTests)
|
||||
ceph-multi-node
|
||||
ceph-single-node
|
||||
ceph-single-node-bluestore
|
||||
ceph-single-node-bluestore-dmcrypt
|
||||
;
|
||||
};
|
||||
};
|
||||
}
|
||||
scope.ceph
|
||||
|
||||
@@ -0,0 +1,89 @@
|
||||
From 20df261e92fd8502398b35456f53761a9bb5ef73 Mon Sep 17 00:00:00 2001
|
||||
From: benaryorg <binary@benary.org>
|
||||
Date: Fri, 20 Mar 2026 02:46:58 +0000
|
||||
Subject: [PATCH 1/2] mgr: python interpreter path fix
|
||||
|
||||
The macro actually created a wide string with the *name* of the constant, not its content.
|
||||
Thus Python failed to find its interpreter instead using `stat(2)` on every element in *PATH* such as `/usr/bin/MGR_PYTHON_EXECUTABLE`.
|
||||
As Python also offers a direct way to hand over the C-string, we can let Python handle the conversion directly, getting rid of the macro entirely.
|
||||
|
||||
Signed-off-by: benaryorg <binary@benary.org>
|
||||
---
|
||||
src/mgr/PyModuleRegistry.cc | 5 ++---
|
||||
1 file changed, 2 insertions(+), 3 deletions(-)
|
||||
|
||||
diff --git a/src/mgr/PyModuleRegistry.cc b/src/mgr/PyModuleRegistry.cc
|
||||
index b340fc64e374929f720eb9ac5d4e7f274707add8..51ab641bf7f7097d0c97ad554880a9cfb1ebd3f2 100644
|
||||
--- a/src/mgr/PyModuleRegistry.cc
|
||||
+++ b/src/mgr/PyModuleRegistry.cc
|
||||
@@ -47,7 +47,6 @@ void PyModuleRegistry::init()
|
||||
std::lock_guard locker(lock);
|
||||
|
||||
// Set up global python interpreter
|
||||
-#define WCHAR(s) L ## #s
|
||||
PyConfig py_config;
|
||||
// do not enable isolated mode, otherwise we would not be able to have access
|
||||
// to the site packages. since we cannot import any module before initializing
|
||||
@@ -68,7 +67,8 @@ void PyModuleRegistry::init()
|
||||
py_config.pathconfig_warnings = 0;
|
||||
|
||||
PyStatus status;
|
||||
- status = PyConfig_SetString(&py_config, &py_config.program_name, WCHAR(MGR_PYTHON_EXECUTABLE));
|
||||
+ status = PyConfig_SetBytesString(&py_config, &py_config.program_name, MGR_PYTHON_EXECUTABLE);
|
||||
+ dout(10) << "set PyConfig program_name to " << std::quoted(MGR_PYTHON_EXECUTABLE) << dendl;
|
||||
ceph_assertf(!PyStatus_Exception(status), "PyConfig_SetString: %s:%s", status.func, status.err_msg);
|
||||
// Some python modules do not cope with an unpopulated argv, so lets
|
||||
// fake one. This step also picks up site-packages into sys.path.
|
||||
@@ -91,7 +91,6 @@ void PyModuleRegistry::init()
|
||||
dout(10) << "set PYTHONPATH to " << std::quoted(pythonpath_env) << dendl;
|
||||
status = Py_InitializeFromConfig(&py_config);
|
||||
ceph_assertf(!PyStatus_Exception(status), "Py_InitializeFromConfig: %s:%s", status.func, status.err_msg);
|
||||
-#undef WCHAR
|
||||
|
||||
// Drop the GIL and remember the main thread state (current
|
||||
// thread state becomes NULL)
|
||||
--
|
||||
2.53.0
|
||||
|
||||
|
||||
From 344d422033cdae87b2bd7768f6bd725b94b2f3bd Mon Sep 17 00:00:00 2001
|
||||
From: benaryorg <binary@benary.org>
|
||||
Date: Fri, 20 Mar 2026 02:49:12 +0000
|
||||
Subject: [PATCH 2/2] cryptotools: use correct interpreter
|
||||
|
||||
Python has support for multiple installed versions and has had so for a long time.
|
||||
The Python version that *ceph-mgr* (for instance) was compiled against may not be the default *python3* from *PATH*.
|
||||
This is the case when testing against a different version, but it is also the case on systems which do not have Python installed globally, or ship the Python version used by Ceph separately (to avoid compatibility issues for instance).
|
||||
|
||||
Luckily *ceph-mgr* has access to the exact Python version and interpreter via CMake already, and it passes this through to Python via the *program_name* setting.
|
||||
This sets the interpreter appropriately and allows us to use that instead.
|
||||
|
||||
Signed-off-by: benaryorg <binary@benary.org>
|
||||
---
|
||||
src/python-common/ceph/cryptotools/remote.py | 3 ++-
|
||||
1 file changed, 2 insertions(+), 1 deletion(-)
|
||||
|
||||
diff --git a/src/python-common/ceph/cryptotools/remote.py b/src/python-common/ceph/cryptotools/remote.py
|
||||
index 2574b4ecdac215624e43312a799564dfed81d4fe..e4efe56466fe1b7e1a786eda2041180aca6186ed 100644
|
||||
--- a/src/python-common/ceph/cryptotools/remote.py
|
||||
+++ b/src/python-common/ceph/cryptotools/remote.py
|
||||
@@ -23,6 +23,7 @@ from typing import List, Union, Dict, Any, Optional, Tuple
|
||||
import json
|
||||
import logging
|
||||
import subprocess
|
||||
+import sys
|
||||
|
||||
from .caller import CryptoCaller, CryptoCallError
|
||||
|
||||
@@ -62,7 +63,7 @@ class ProcessCryptoCaller(CryptoCaller):
|
||||
_input = None
|
||||
else:
|
||||
_input = input_data.encode()
|
||||
- cmd = ['python3', '-m', _ctmodule] + list(args)
|
||||
+ cmd = [sys.executable, '-m', _ctmodule] + list(args)
|
||||
logger.warning('CryptoCaller will run: %r', cmd)
|
||||
try:
|
||||
return subprocess.run(
|
||||
--
|
||||
2.53.0
|
||||
|
||||
@@ -0,0 +1,34 @@
|
||||
{
|
||||
ceph-python,
|
||||
ceph-meta,
|
||||
ceph-src,
|
||||
}:
|
||||
|
||||
ceph-python.pkgs.buildPythonPackage {
|
||||
pname = "ceph-common";
|
||||
inherit (ceph-src) version;
|
||||
src = ceph-src;
|
||||
|
||||
pyproject = true;
|
||||
|
||||
sourceRoot = "${ceph-src.name}/src/python-common";
|
||||
|
||||
build-system = with ceph-python.pkgs; [
|
||||
setuptools
|
||||
];
|
||||
|
||||
dependencies = with ceph-python.pkgs; [
|
||||
pyyaml
|
||||
];
|
||||
|
||||
nativeCheckInputs = with ceph-python.pkgs; [
|
||||
pytestCheckHook
|
||||
];
|
||||
|
||||
disabledTests = [
|
||||
# requires network access
|
||||
"test_valid_addr"
|
||||
];
|
||||
|
||||
meta = ceph-meta "Ceph common module for code shared by manager modules";
|
||||
}
|
||||
@@ -0,0 +1,50 @@
|
||||
{
|
||||
ceph-python,
|
||||
ceph-python-common,
|
||||
}:
|
||||
|
||||
# TODO: split this off in build and runtime environment
|
||||
ceph-python.withPackages (
|
||||
ps: with ps; [
|
||||
ceph-python-common
|
||||
|
||||
# build time
|
||||
cython_0
|
||||
|
||||
# debian/control
|
||||
bcrypt
|
||||
cherrypy
|
||||
influxdb
|
||||
jinja2
|
||||
jmespath
|
||||
kubernetes
|
||||
natsort
|
||||
numpy
|
||||
pecan
|
||||
prettytable
|
||||
pyjwt
|
||||
pyopenssl
|
||||
python-dateutil
|
||||
pyyaml
|
||||
requests
|
||||
routes
|
||||
scikit-learn
|
||||
scipy
|
||||
setuptools
|
||||
sphinx
|
||||
virtualenv
|
||||
werkzeug
|
||||
xmltodict
|
||||
|
||||
# src/cephadm/zipapp-reqs.txt
|
||||
markupsafe
|
||||
|
||||
# src/pybind/mgr/requirements-required.txt
|
||||
cryptography
|
||||
jsonpatch
|
||||
|
||||
# src/tools/cephfs/shell/setup.py
|
||||
cmd2
|
||||
colorama
|
||||
]
|
||||
)
|
||||
@@ -0,0 +1,14 @@
|
||||
{
|
||||
rocksdb,
|
||||
fetchFromGitHub,
|
||||
}:
|
||||
|
||||
rocksdb.overrideAttrs rec {
|
||||
version = "7.9.2";
|
||||
src = fetchFromGitHub {
|
||||
owner = "facebook";
|
||||
repo = "rocksdb";
|
||||
tag = "v${version}";
|
||||
hash = "sha256-5P7IqJ14EZzDkbjaBvbix04ceGGdlWBuVFH/5dpD5VM=";
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,32 @@
|
||||
{ lib, pkgs }:
|
||||
|
||||
# the README.md in the same directory contains further information on why this requires a scope
|
||||
lib.makeScope pkgs.newScope (self: {
|
||||
ceph-rocksdb = self.callPackage ./rocksdb.nix { };
|
||||
|
||||
# to get an idea which Python versions are supported by Ceph, see upstream `do_cmake.sh` (see `PYBUILD=` variable)
|
||||
ceph-python = self.callPackage ({ python312 }: python312) { };
|
||||
ceph-python-common = self.callPackage ./python-common.nix { };
|
||||
ceph-python-env = self.callPackage ./python-env.nix { };
|
||||
|
||||
# Note when trying to upgrade boost:
|
||||
# * When upgrading Ceph, it's recommended to check which boost version Ceph uses on Fedora,
|
||||
# and default to that.
|
||||
# * The version that Ceph downloads if `-DWITH_SYSTEM_BOOST:BOOL=ON` is not given
|
||||
# is declared in `cmake/modules/BuildBoost.cmake` line `set(boost_version ...)`.
|
||||
ceph-boost = self.callPackage (
|
||||
{ boost187, ceph-python }:
|
||||
boost187.override {
|
||||
enablePython = true;
|
||||
python = ceph-python;
|
||||
}
|
||||
) { };
|
||||
|
||||
# not actually a derivation, just a DRY helper
|
||||
ceph-meta = self.callPackage ./meta.nix { };
|
||||
|
||||
# patched source to build all derivations from
|
||||
ceph-src = self.callPackage ./src.nix { };
|
||||
|
||||
ceph = self.callPackage ./ceph.nix { };
|
||||
})
|
||||
@@ -0,0 +1,30 @@
|
||||
{
|
||||
applyPatches,
|
||||
fetchpatch2,
|
||||
fetchurl,
|
||||
}:
|
||||
|
||||
applyPatches (final: {
|
||||
pname = "ceph-src";
|
||||
version = "20.2.1";
|
||||
|
||||
src = fetchurl {
|
||||
url = "https://download.ceph.com/tarballs/ceph-${final.version}.tar.gz";
|
||||
hash = "sha256-3neaoBQYOTiLsgHgqdYiuEM5guHE17/DrGEXt2OXJUI=";
|
||||
};
|
||||
|
||||
patches = [
|
||||
# required to be able to compile s3select against nixpkgs' arrow-cpp
|
||||
# See: https://github.com/ceph/s3select/pull/169
|
||||
(fetchpatch2 {
|
||||
name = "ceph-s3select-arrow-cpp-20.patch";
|
||||
url = "https://github.com/ceph/s3select/pull/169.diff?full_index=1";
|
||||
extraPrefix = "src/s3select/";
|
||||
stripLen = 1;
|
||||
hash = "sha256-0jn5X4jIdluCufFXWHeO6skMz6XQpliHkC1tPLK6dbk=";
|
||||
})
|
||||
# fixes issues when python3 is not on the PATH
|
||||
# See: https://github.com/ceph/ceph/pull/67904
|
||||
./patches/0001-mgr-python-interpreter.patch
|
||||
];
|
||||
})
|
||||
Reference in New Issue
Block a user