From e688a60e6991df7cc247c67bda3cd14350b035cd Mon Sep 17 00:00:00 2001 From: Raito Bezarius Date: Sun, 24 Mar 2024 15:10:09 +0100 Subject: [PATCH] netdata: 1.44.3 -> 1.45.0 https://github.com/netdata/netdata/releases/tag/v1.45.0 Netdata moved to CMake and required an overhaul of the whole expression. `netdata-go-plugins` has been moved back in the monorepo, leading to a removal of the standalone Go plugin expression. The eBPF plugin is broken for now. The patch for disabling the dashboard v2 has been upgraded and sent to upstream again. ACLK is correctly rendered optional now. This adds the network-viewer, logs-management and debugfs plugin support. Signed-off-by: Raito Bezarius --- pkgs/tools/system/netdata/default.nix | 138 ++++++++++++------ pkgs/tools/system/netdata/go.d.plugin.nix | 35 ----- .../netdata/no-files-in-etc-and-var.patch | 128 ---------------- .../netdata/skip-CONFIGURE_COMMAND.patch | 16 -- pkgs/top-level/all-packages.nix | 2 - 5 files changed, 95 insertions(+), 224 deletions(-) delete mode 100644 pkgs/tools/system/netdata/go.d.plugin.nix delete mode 100644 pkgs/tools/system/netdata/no-files-in-etc-and-var.patch delete mode 100644 pkgs/tools/system/netdata/skip-CONFIGURE_COMMAND.patch diff --git a/pkgs/tools/system/netdata/default.nix b/pkgs/tools/system/netdata/default.nix index b2461760da70..775a425edb91 100644 --- a/pkgs/tools/system/netdata/default.nix +++ b/pkgs/tools/system/netdata/default.nix @@ -1,9 +1,9 @@ -{ lib, stdenv, fetchFromGitHub, fetchpatch, autoreconfHook, pkg-config, makeWrapper +{ lib, stdenv, fetchFromGitHub, fetchpatch, cmake, pkg-config, makeWrapper , CoreFoundation, IOKit, libossp_uuid , nixosTests -, netdata-go-plugins -, bash, curl, jemalloc, json_c, libuv, zlib, libyaml +, bash, curl, jemalloc, json_c, libuv, zlib, libyaml, libelf, libbpf , libcap, libuuid, lm_sensors, protobuf +, go, buildGoModule, ninja , withCups ? false, cups , withDBengine ? true, lz4 , withIpmi ? (!stdenv.isDarwin), freeipmi @@ -15,11 +15,13 @@ , withSsl ? true, openssl , withSystemdJournal ? (!stdenv.isDarwin), systemd , withDebug ? false +, withEbpf ? false +, withNetworkViewer ? (!stdenv.isDarwin) }: stdenv.mkDerivation rec { # Don't forget to update go.d.plugin.nix as well - version = "1.44.3"; + version = "1.45.0"; pname = "netdata"; src = fetchFromGitHub { @@ -27,21 +29,21 @@ stdenv.mkDerivation rec { repo = "netdata"; rev = "v${version}"; hash = if withCloudUi - then "sha256-ahWaq6geEoc6NZ2oU/Dqnb0bjRXd+q1zaRGOSIYVYok=" - else "sha256-2Kvh2WuoJjJxsFKueMjCAbazqZdzoOTxakbPVsj9PBo="; + then "sha256-AyhS5vwGZLnc+xmf9Qvk2v8rTzhHSUe7LpAbwRYjrKM=" + else "sha256-11UHXPnXC4pzK/oXOZJ7f9X86EV9uHOvMtOCnxTV86U="; fetchSubmodules = true; # Remove v2 dashboard distributed under NCUL1. Make sure an empty # Makefile.am exists, as autoreconf will get confused otherwise. postFetch = lib.optionalString (!withCloudUi) '' - rm -rf $out/web/gui/v2/* - touch $out/web/gui/v2/Makefile.am + rm -rf $out/src/web/gui/v2/* + touch $out/src/web/gui/v2/Makefile.am ''; }; strictDeps = true; - nativeBuildInputs = [ autoreconfHook pkg-config makeWrapper protobuf ]; + nativeBuildInputs = [ cmake pkg-config makeWrapper go ninja ]; # bash is only used to rewrite shebangs buildInputs = [ bash curl jemalloc json_c libuv zlib libyaml ] ++ lib.optionals stdenv.isDarwin [ CoreFoundation IOKit libossp_uuid ] @@ -52,23 +54,16 @@ stdenv.mkDerivation rec { ++ lib.optionals withNetfilter [ libmnl libnetfilter_acct ] ++ lib.optionals withConnPubSub [ google-cloud-cpp grpc ] ++ lib.optionals withConnPrometheus [ snappy ] + ++ lib.optionals withEbpf [ libelf libbpf ] ++ lib.optionals (withCloud || withConnPrometheus) [ protobuf ] ++ lib.optionals withSystemdJournal [ systemd ] ++ lib.optionals withSsl [ openssl ]; patches = [ - # required to prevent plugins from relying on /etc - # and /var - ./no-files-in-etc-and-var.patch - - # Avoid build-only inputs in closure leaked by configure command: - # https://github.com/NixOS/nixpkgs/issues/175693#issuecomment-1143344162 - ./skip-CONFIGURE_COMMAND.patch - # Allow building without non-free v2 dashboard. (fetchpatch { - url = "https://github.com/peat-psuwit/netdata/commit/6ccbdd1500db2b205923968688d5f1777430a326.patch"; - hash = "sha256-jAyk5HlxdjFn5IP6jOKP8/SXOraMQSA6r1krThe+s7g="; + url = "https://github.com/netdata/netdata/pull/17240/commits/b108df72281633234b731b223d99ec99f1d36adf.patch"; + hash = "sha256-tgsnbNY0pxFU3bz1J1qPaAeVsozsk2bpHV2mNy8A9is="; }) ]; @@ -83,8 +78,8 @@ stdenv.mkDerivation rec { env.NIX_CFLAGS_COMPILE = lib.optionalString withDebug "-O1 -ggdb -DNETDATA_INTERNAL_CHECKS=1"; postInstall = '' - ln -s ${netdata-go-plugins}/lib/netdata/conf.d/* $out/lib/netdata/conf.d - ln -s ${netdata-go-plugins}/bin/godplugin $out/libexec/netdata/plugins.d/go.d.plugin + # Relocate one folder above. + mv $out/usr/* $out/ '' + lib.optionalString (!stdenv.isDarwin) '' # rename this plugin so netdata will look for setuid wrapper mv $out/libexec/netdata/plugins.d/apps.plugin \ @@ -95,6 +90,10 @@ stdenv.mkDerivation rec { $out/libexec/netdata/plugins.d/perf.plugin.org mv $out/libexec/netdata/plugins.d/slabinfo.plugin \ $out/libexec/netdata/plugins.d/slabinfo.plugin.org + mv $out/libexec/netdata/plugins.d/debugfs.plugin \ + $out/libexec/netdata/plugins.d/debugfs.plugin.org + mv $out/libexec/netdata/plugins.d/logs-management.plugin \ + $out/libexec/netdata/plugins.d/logs-management.plugin.org ${lib.optionalString withSystemdJournal '' mv $out/libexec/netdata/plugins.d/systemd-journal.plugin \ $out/libexec/netdata/plugins.d/systemd-journal.plugin.org @@ -103,47 +102,100 @@ stdenv.mkDerivation rec { mv $out/libexec/netdata/plugins.d/freeipmi.plugin \ $out/libexec/netdata/plugins.d/freeipmi.plugin.org ''} + ${lib.optionalString withNetworkViewer '' + mv $out/libexec/netdata/plugins.d/network-viewer.plugin \ + $out/libexec/netdata/plugins.d/network-viewer.plugin.org + ''} ''; preConfigure = lib.optionalString (!stdenv.isDarwin) '' - substituteInPlace collectors/python.d.plugin/python_modules/third_party/lm_sensors.py \ - --replace 'ctypes.util.find_library("sensors")' '"${lm_sensors.out}/lib/libsensors${stdenv.hostPlatform.extensions.sharedLibrary}"' + substituteInPlace src/collectors/python.d.plugin/python_modules/third_party/lm_sensors.py \ + --replace-fail 'ctypes.util.find_library("sensors")' '"${lm_sensors.out}/lib/libsensors${stdenv.hostPlatform.extensions.sharedLibrary}"' + '' + '' + export GOCACHE=$TMPDIR/go-cache + export GOPATH=$TMPDIR/go + export GOPROXY=file://${passthru.netdata-go-modules} + export GOSUMDB=off + + # Prevent the path to be caught into the Nix store path. + substituteInPlace CMakeLists.txt \ + --replace-fail 'set(CACHE_DIR "''${CMAKE_INSTALL_PREFIX}/var/cache/netdata")' 'set(CACHE_DIR "/var/cache/netdata")' \ + --replace-fail 'set(CONFIG_DIR "''${CMAKE_INSTALL_PREFIX}/etc/netdata")' 'set(CONFIG_DIR "/etc/netdata")' \ + --replace-fail 'set(LIBCONFIG_DIR "''${CMAKE_INSTALL_PREFIX}/usr/lib/netdata/conf.d")' 'set(LIBCONFIG_DIR "${placeholder "out"}/share/netdata/conf.d")' \ + --replace-fail 'set(LOG_DIR "''${CMAKE_INSTALL_PREFIX}/var/log/netdata")' 'set(LOG_DIR "/var/log/netdata")' \ + --replace-fail 'set(PLUGINS_DIR "''${CMAKE_INSTALL_PREFIX}/usr/libexec/netdata/plugins.d")' 'set(PLUGINS_DIR "${placeholder "out"}/libexec/netdata/plugins.d")' \ + --replace-fail 'set(VARLIB_DIR "''${CMAKE_INSTALL_PREFIX}/var/lib/netdata")' 'set(VARLIB_DIR "/var/lib/netdata")' \ + --replace-fail 'set(pkglibexecdir_POST "''${CMAKE_INSTALL_PREFIX}/usr/libexec/netdata")' 'set(pkglibexecdir_POST "${placeholder "out"}/libexec/netdata")' \ + --replace-fail 'set(localstatedir_POST "''${CMAKE_INSTALL_PREFIX}/var")' 'set(localstatedir_POST "/var")' \ + --replace-fail 'set(sbindir_POST "''${CMAKE_INSTALL_PREFIX}/usr/sbin")' 'set(sbindir_POST "${placeholder "out"}/bin")' \ + --replace-fail 'set(configdir_POST "''${CMAKE_INSTALL_PREFIX}/etc/netdata")' 'set(configdir_POST "/etc/netdata")' \ + --replace-fail 'set(libconfigdir_POST "''${CMAKE_INSTALL_PREFIX}/usr/lib/netdata/conf.d")' 'set(libconfigdir_POST "${placeholder "out"}/share/netdata/conf.d")' \ + --replace-fail 'set(cachedir_POST "''${CMAKE_INSTALL_PREFIX}/var/cache/netdata")' 'set(libconfigdir_POST "/var/cache/netdata")' \ + --replace-fail 'set(registrydir_POST "''${CMAKE_INSTALL_PREFIX}/var/lib/netdata/registry")' 'set(registrydir_POST "/var/lib/netdata/registry")' \ + --replace-fail 'set(varlibdir_POST "''${CMAKE_INSTALL_PREFIX}/var/lib/netdata")' 'set(varlibdir_POST "/var/lib/netdata")' ''; - configureFlags = [ - "--localstatedir=/var" - "--sysconfdir=/etc" - "--disable-ebpf" - "--with-jemalloc=${jemalloc}" - ] ++ lib.optionals (withSystemdJournal) [ - "--enable-plugin-systemd-journal" - ] ++ lib.optionals (!withDBengine) [ - "--disable-dbengine" - ] ++ lib.optionals (!withCloud) [ - "--disable-cloud" - ] ++ lib.optionals (!withCloudUi) [ - "--disable-cloud-ui" + cmakeFlags = [ + "-DWEB_DIR=share/netdata/web" + (lib.cmakeBool "ENABLE_CLOUD" withCloud) + # ACLK is agent cloud link. + (lib.cmakeBool "ENABLE_ACLK" withCloud) + (lib.cmakeBool "ENABLE_DASHBOARD_V2" withCloudUi) + (lib.cmakeBool "ENABLE_DBENGINE" withDBengine) + (lib.cmakeBool "ENABLE_PLUGIN_FREEIPMI" withIpmi) + (lib.cmakeBool "ENABLE_PLUGIN_SYSTEMD_JOURNAL" withSystemdJournal) + (lib.cmakeBool "ENABLE_PLUGIN_NETWORK_VIEWER" withNetworkViewer) + (lib.cmakeBool "ENABLE_PLUGIN_EBPF" withEbpf) + # raitobezarius: I do not wish to maintain Xen-related things alone, thus, disabled. + # Feel free to open an issue / PR to fix this. + (lib.cmakeBool "ENABLE_PLUGIN_XENSTAT" false) + (lib.cmakeBool "ENABLE_PLUGIN_CUPS" withCups) + (lib.cmakeBool "ENABLE_EXPORTER_PROMETHEUS_REMOTE_WRITE" withConnPrometheus) + (lib.cmakeBool "ENABLE_JEMALLOC" true) + # Suggested by upstream. + "-G Ninja" ]; postFixup = '' - # remove once https://github.com/netdata/netdata/pull/16300 merged - substituteInPlace $out/bin/netdata-claim.sh \ - --replace /bin/echo echo - wrapProgram $out/bin/netdata-claim.sh --prefix PATH : ${lib.makeBinPath [ openssl ]} wrapProgram $out/libexec/netdata/plugins.d/cgroup-network-helper.sh --prefix PATH : ${lib.makeBinPath [ bash ]} wrapProgram $out/bin/netdatacli --set NETDATA_PIPENAME /run/netdata/ipc + + # Time to cleanup the output directory. + unlink $out/sbin + cp $out/etc/netdata/edit-config $out/bin/netdata-edit-config + mv $out/lib/netdata/conf.d $out/share/netdata/conf.d + rm -rf $out/{var,usr,etc} ''; enableParallelBuild = true; - passthru = { - inherit withIpmi; + passthru = rec { + netdata-go-modules = (buildGoModule { + pname = "netdata-go-plugins"; + inherit version src; + + sourceRoot = "${src.name}/src/go/collectors/go.d.plugin"; + + vendorHash = "sha256-KO+xMk6fpZCYRyxxKrsGfOHJ2bwjBaSmkgz1jIUHaZs="; + doCheck = false; + proxyVendor = true; + + ldflags = [ "-s" "-w" "-X main.version=${version}" ]; + + passthru.tests = tests; + meta = meta // { + description = "Netdata orchestrator for data collection modules written in Go"; + mainProgram = "godplugin"; + license = lib.licenses.gpl3Only; + }; + }).goModules; + inherit withIpmi withNetworkViewer; tests.netdata = nixosTests.netdata; }; meta = with lib; { - broken = stdenv.isDarwin || stdenv.buildPlatform != stdenv.hostPlatform; + broken = stdenv.isDarwin || stdenv.buildPlatform != stdenv.hostPlatform || withEbpf; description = "Real-time performance monitoring tool"; homepage = "https://www.netdata.cloud/"; changelog = "https://github.com/netdata/netdata/releases/tag/v${version}"; diff --git a/pkgs/tools/system/netdata/go.d.plugin.nix b/pkgs/tools/system/netdata/go.d.plugin.nix deleted file mode 100644 index 910ad29c13fa..000000000000 --- a/pkgs/tools/system/netdata/go.d.plugin.nix +++ /dev/null @@ -1,35 +0,0 @@ -{ lib, fetchFromGitHub, buildGoModule, nixosTests }: - -buildGoModule rec { - pname = "netdata-go-plugins"; - version = "0.58.1"; - - src = fetchFromGitHub { - owner = "netdata"; - repo = "go.d.plugin"; - rev = "v${version}"; - hash = "sha256-zzHm98jec7MXnzVsrLlYIk+ILA3Ei43853dM1LdFz5c="; - }; - - vendorHash = "sha256-eb+GRFhfWxDkfH4x2VF3ogyT5z4OcIoqHtEVJ1tGsdA="; - - doCheck = false; - - ldflags = [ "-s" "-w" "-X main.version=${version}" ]; - - postInstall = '' - mkdir -p $out/lib/netdata/conf.d - cp -r config/* $out/lib/netdata/conf.d - ''; - - passthru.tests = { inherit (nixosTests) netdata; }; - - meta = with lib; { - description = "Netdata orchestrator for data collection modules written in go"; - mainProgram = "godplugin"; - homepage = "https://github.com/netdata/go.d.plugin"; - changelog = "https://github.com/netdata/go.d.plugin/releases/tag/v${version}"; - license = licenses.gpl3Only; - maintainers = [ maintainers.raitobezarius ]; - }; -} diff --git a/pkgs/tools/system/netdata/no-files-in-etc-and-var.patch b/pkgs/tools/system/netdata/no-files-in-etc-and-var.patch deleted file mode 100644 index 039376fe4e3a..000000000000 --- a/pkgs/tools/system/netdata/no-files-in-etc-and-var.patch +++ /dev/null @@ -1,128 +0,0 @@ -diff --git c/collectors/Makefile.am i/collectors/Makefile.am -index 1bbb2e0ef..96c400d33 100644 ---- c/collectors/Makefile.am -+++ i/collectors/Makefile.am -@@ -33,7 +33,7 @@ usercustompluginsconfigdir=$(configdir)/custom-plugins.d - usergoconfigdir=$(configdir)/go.d - - # Explicitly install directories to avoid permission issues due to umask --install-exec-local: -+no-install-exec-local: - $(INSTALL) -d $(DESTDIR)$(usercustompluginsconfigdir) - $(INSTALL) -d $(DESTDIR)$(usergoconfigdir) - -diff --git c/collectors/charts.d.plugin/Makefile.am i/collectors/charts.d.plugin/Makefile.am -index f82992fd4..4cac1ae4f 100644 ---- c/collectors/charts.d.plugin/Makefile.am -+++ i/collectors/charts.d.plugin/Makefile.am -@@ -34,7 +34,7 @@ dist_userchartsconfig_DATA = \ - $(NULL) - - # Explicitly install directories to avoid permission issues due to umask --install-exec-local: -+no-install-exec-local: - $(INSTALL) -d $(DESTDIR)$(userchartsconfigdir) - - chartsconfigdir=$(libconfigdir)/charts.d -diff --git c/collectors/ebpf.plugin/Makefile.am i/collectors/ebpf.plugin/Makefile.am -index 2d5f92a6b..8b11c7502 100644 ---- c/collectors/ebpf.plugin/Makefile.am -+++ i/collectors/ebpf.plugin/Makefile.am -@@ -9,7 +9,7 @@ SUFFIXES = .in - userebpfconfigdir=$(configdir)/ebpf.d - - # Explicitly install directories to avoid permission issues due to umask --install-exec-local: -+no-install-exec-local: - $(INSTALL) -d $(DESTDIR)$(userebpfconfigdir) - - dist_noinst_DATA = \ -diff --git c/collectors/python.d.plugin/Makefile.am i/collectors/python.d.plugin/Makefile.am -index ca49c1c02..1b9bcc446 100644 ---- c/collectors/python.d.plugin/Makefile.am -+++ i/collectors/python.d.plugin/Makefile.am -@@ -32,7 +32,7 @@ dist_userpythonconfig_DATA = \ - $(NULL) - - # Explicitly install directories to avoid permission issues due to umask --install-exec-local: -+no-install-exec-local: - $(INSTALL) -d $(DESTDIR)$(userpythonconfigdir) - - pythonconfigdir=$(libconfigdir)/python.d -diff --git c/collectors/statsd.plugin/Makefile.am i/collectors/statsd.plugin/Makefile.am -index c8144c137..f8aaa89b6 100644 ---- c/collectors/statsd.plugin/Makefile.am -+++ i/collectors/statsd.plugin/Makefile.am -@@ -19,5 +19,5 @@ dist_userstatsdconfig_DATA = \ - $(NULL) - - # Explicitly install directories to avoid permission issues due to umask --install-exec-local: -+no-install-exec-local: - $(INSTALL) -d $(DESTDIR)$(userstatsdconfigdir) -diff --git c/health/Makefile.am i/health/Makefile.am -index 7d7bca4cc..3086876dd 100644 ---- c/health/Makefile.am -+++ i/health/Makefile.am -@@ -19,7 +19,7 @@ dist_userhealthconfig_DATA = \ - $(NULL) - - # Explicitly install directories to avoid permission issues due to umask --install-exec-local: -+no-install-exec-local: - $(INSTALL) -d $(DESTDIR)$(userhealthconfigdir) - - healthconfigdir=$(libconfigdir)/health.d -diff --git c/logsmanagement/Makefile.am i/logsmanagement/Makefile.am -index 33f08d556..1f08cbae9 100644 ---- c/logsmanagement/Makefile.am -+++ i/logsmanagement/Makefile.am -@@ -6,7 +6,7 @@ MAINTAINERCLEANFILES = $(srcdir)/Makefile.in - userlogsmanagconfigdir=$(configdir)/logsmanagement.d - - # Explicitly install directories to avoid permission issues due to umask --install-exec-local: -+no-install-exec-local: - $(INSTALL) -d $(DESTDIR)$(userlogsmanagconfigdir) - - dist_libconfig_DATA = \ -diff --git c/system/Makefile.am i/system/Makefile.am -index 1e96f6f4f..98122ecdc 100644 ---- c/system/Makefile.am -+++ i/system/Makefile.am -@@ -22,12 +22,9 @@ include $(top_srcdir)/build/subst.inc - SUFFIXES = .in - - dist_config_SCRIPTS = \ -- edit-config \ - $(NULL) - - dist_config_DATA = \ -- .install-type \ -- netdata-updater.conf \ - $(NULL) - - libconfigvnodesdir=$(libconfigdir)/vnodes -@@ -47,7 +44,7 @@ libsysrunitdir=$(libsysdir)/runit - libsyssystemddir=$(libsysdir)/systemd - - # Explicitly install directories to avoid permission issues due to umask --install-exec-local: -+no-install-exec-local: - $(INSTALL) -d $(DESTDIR)$(configdir) - $(INSTALL) -d $(DESTDIR)$(libsysdir) - $(INSTALL) -d $(DESTDIR)$(libsyscrondir) -diff --git c/web/Makefile.am i/web/Makefile.am -index be2c545c3..55f373114 100644 ---- c/web/Makefile.am -+++ i/web/Makefile.am -@@ -13,7 +13,7 @@ SUBDIRS = \ - usersslconfigdir=$(configdir)/ssl - - # Explicitly install directories to avoid permission issues due to umask --install-exec-local: -+no-install-exec-local: - $(INSTALL) -d $(DESTDIR)$(usersslconfigdir) - - dist_noinst_DATA = \ diff --git a/pkgs/tools/system/netdata/skip-CONFIGURE_COMMAND.patch b/pkgs/tools/system/netdata/skip-CONFIGURE_COMMAND.patch deleted file mode 100644 index 472da270798f..000000000000 --- a/pkgs/tools/system/netdata/skip-CONFIGURE_COMMAND.patch +++ /dev/null @@ -1,16 +0,0 @@ -Shrink closure size by avoiding paths embedded from configure call. - -https://github.com/NixOS/nixpkgs/issues/175693 -diff --git a/daemon/buildinfo.c b/daemon/buildinfo.c -index 56cde84fc..011e7579d 100644 ---- a/daemon/buildinfo.c -+++ b/daemon/buildinfo.c -@@ -1040,7 +1040,7 @@ static void build_info_set_status(BUILD_INFO_SLOT slot, bool status) { - - __attribute__((constructor)) void initialize_build_info(void) { - build_info_set_value(BIB_PACKAGING_NETDATA_VERSION, program_version); -- build_info_set_value(BIB_PACKAGING_CONFIGURE_OPTIONS, CONFIGURE_COMMAND); -+ build_info_set_value(BIB_PACKAGING_CONFIGURE_OPTIONS, "REMOVED FOR CLOSURE SIZE REASONS"); - - #ifdef COMPILED_FOR_LINUX - build_info_set_status(BIB_FEATURE_BUILT_FOR, true); diff --git a/pkgs/top-level/all-packages.nix b/pkgs/top-level/all-packages.nix index e3eb44f6b418..964d2a574cb7 100644 --- a/pkgs/top-level/all-packages.nix +++ b/pkgs/top-level/all-packages.nix @@ -10094,8 +10094,6 @@ with pkgs; withCloud = !stdenv.isDarwin; withCloudUi = true; }; - # Exposed here so the bots can auto-upgrade it - netdata-go-plugins = callPackage ../tools/system/netdata/go.d.plugin.nix { }; netsurf = recurseIntoAttrs (callPackage ../applications/networking/browsers/netsurf { }); netsurf-browser = netsurf.browser;