Merge pull request #201562 from jdbaldry/add/osquery

maintainers/maintainer-list.nix

@@ -18466,6 +18466,12 @@
     github = "zmitchell";
     githubId = 10246891;
   };
+  znewman01 = {
+    email = "znewman01@gmail.com";
+    github = "znewman01";
+    githubId = 873857;
+    name = "Zack Newman";
+  };
   zoedsoupe = {
     github = "zoedsoupe";
     githubId = 44469426;

nixos/doc/manual/release-notes/rl-2311.section.md

@@ -26,6 +26,8 @@
 
 - [trust-dns](https://trust-dns.org/), a Rust based DNS server built to be safe and secure from the ground up. Available as [services.trust-dns](#opt-services.trust-dns.enable).
 
+- [osquery](https://www.osquery.io/), a SQL powered operating system instrumentation, monitoring, and analytics.
+
 ## Backward Incompatibilities {#sec-release-23.11-incompatibilities}
 
 - The `boot.loader.raspberryPi` options have been marked deprecated, with intent for removal for NixOS 24.11. They had a limited use-case, and do not work like people expect. They required either very old installs ([before mid-2019](https://github.com/NixOS/nixpkgs/pull/62462)) or customized builds out of scope of the standard and generic AArch64 support. That option set never supported the Raspberry Pi 4 family of devices.

nixos/modules/module-list.nix

@@ -764,6 +764,7 @@
   ./services/monitoring/nagios.nix
   ./services/monitoring/netdata.nix
   ./services/monitoring/opentelemetry-collector.nix
+  ./services/monitoring/osquery.nix
   ./services/monitoring/parsedmarc.nix
   ./services/monitoring/prometheus/alertmanager-irc-relay.nix
   ./services/monitoring/prometheus/alertmanager.nix

nixos/modules/rename.nix

@@ -72,7 +72,6 @@ in
     (mkRemovedOptionModule [ "services" "mesos" ] "The corresponding package was removed from nixpkgs.")
     (mkRemovedOptionModule [ "services" "moinmoin" ] "The corresponding package was removed from nixpkgs.")
     (mkRemovedOptionModule [ "services" "mwlib" ] "The corresponding package was removed from nixpkgs.")
-    (mkRemovedOptionModule [ "services" "osquery" ] "The osquery module has been removed")
     (mkRemovedOptionModule [ "services" "pantheon" "files" ] ''
       This module was removed, please add pkgs.pantheon.elementary-files to environment.systemPackages directly.
     '')

nixos/modules/services/monitoring/osquery.nix

@@ -0,0 +1,97 @@
+{ config, lib, pkgs, ... }:
+
+with lib;
+let
+  cfg = config.services.osquery;
+  dirname = path: with lib.strings; with lib.lists; concatStringsSep "/"
+    (init (splitString "/" (normalizePath path)));
+
+  # conf is the osquery configuration file used when the --config_plugin=filesystem.
+  # filesystem is the osquery default value for the config_plugin flag.
+  conf = pkgs.writeText "osquery.conf" (builtins.toJSON cfg.settings);
+
+  # flagfile is the file containing osquery command line flags to be
+  # provided to the application using the special --flagfile option.
+  flagfile = pkgs.writeText "osquery.flags"
+    (concatStringsSep "\n"
+      (mapAttrsToList (name: value: "--${name}=${value}")
+        # Use the conf derivation if not otherwise specified.
+        ({ config_path = conf; } // cfg.flags)));
+
+  osqueryi = pkgs.runCommand "osqueryi" { nativeBuildInputs = [ pkgs.makeWrapper ]; } ''
+    mkdir -p $out/bin
+    makeWrapper ${pkgs.osquery}/bin/osqueryi $out/bin/osqueryi \
+      --add-flags "--flagfile ${flagfile}"
+  '';
+in
+{
+  options.services.osquery = {
+    enable = mkEnableOption (mdDoc "osqueryd daemon");
+
+    settings = mkOption {
+      default = { };
+      description = mdDoc ''
+        Configuration to be written to the osqueryd JSON configuration file.
+        To understand the configuration format, refer to https://osquery.readthedocs.io/en/stable/deployment/configuration/#configuration-components.
+      '';
+      example = {
+        options.utc = false;
+      };
+      type = types.attrs;
+    };
+
+    flags = mkOption {
+      default = { };
+      description = mdDoc ''
+        Attribute set of flag names and values to be written to the osqueryd flagfile.
+        For more information, refer to https://osquery.readthedocs.io/en/stable/installation/cli-flags.
+      '';
+      example = {
+        config_refresh = "10";
+      };
+      type = with types;
+        submodule {
+          freeformType = attrsOf str;
+          options = {
+            database_path = mkOption {
+              default = "/var/lib/osquery/osquery.db";
+              readOnly = true;
+              description = mdDoc "Path used for the database file.";
+              type = path;
+            };
+            logger_path = mkOption {
+              default = "/var/log/osquery";
+              readOnly = true;
+              description = mdDoc "Base directory used for logging.";
+              type = path;
+            };
+            pidfile = mkOption {
+              default = "/run/osquery/osqueryd.pid";
+              readOnly = true;
+              description = mdDoc "Path used for pid file.";
+              type = path;
+            };
+          };
+        };
+    };
+  };
+
+  config = mkIf cfg.enable {
+    environment.systemPackages = [ osqueryi ];
+    systemd.services.osqueryd = {
+      after = [ "network.target" "syslog.service" ];
+      description = "The osquery daemon";
+      serviceConfig = {
+        ExecStart = "${pkgs.osquery}/bin/osqueryd --flagfile ${flagfile}";
+        PIDFile = cfg.flags.pidfile;
+        LogsDirectory = cfg.flags.logger_path;
+        StateDirectory = dirname cfg.flags.database_path;
+        Restart = "always";
+      };
+      wantedBy = [ "multi-user.target" ];
+    };
+    systemd.tmpfiles.rules = [
+      "d ${dirname (cfg.flags.pidfile)} 0755 root root -"
+    ];
+  };
+}

nixos/tests/all-tests.nix

@@ -577,6 +577,7 @@ in {
   openvscode-server = handleTest ./openvscode-server.nix {};
   orangefs = handleTest ./orangefs.nix {};
   os-prober = handleTestOn ["x86_64-linux"] ./os-prober.nix {};
+  osquery = handleTestOn ["x86_64-linux"] ./osquery.nix {};
   osrm-backend = handleTest ./osrm-backend.nix {};
   overlayfs = handleTest ./overlayfs.nix {};
   pacemaker = handleTest ./pacemaker.nix {};

nixos/tests/osquery.nix

@@ -0,0 +1,56 @@
+import ./make-test-python.nix ({ lib, pkgs, ... }:
+
+with lib;
+
+let
+  config_refresh = "10";
+  nullvalue = "NULL";
+  utc = false;
+in
+{
+  name = "osquery";
+  meta = with maintainers; {
+    maintainers = [ znewman01 lewo ];
+  };
+
+  nodes.machine = { config, pkgs, ... }: {
+    services.osquery = {
+      enable = true;
+
+      settings.options = { inherit nullvalue utc; };
+      flags = {
+        inherit config_refresh;
+        nullvalue = "IGNORED";
+      };
+    };
+  };
+
+  testScript = { nodes, ... }:
+    let
+      cfg = nodes.machine.services.osquery;
+    in
+    ''
+      machine.start()
+      machine.wait_for_unit("osqueryd.service")
+
+      # Stop the osqueryd service so that we can use osqueryi to check information stored in the database.
+      machine.wait_until_succeeds("systemctl stop osqueryd.service")
+
+      # osqueryd was able to query information about the host.
+      machine.succeed("echo 'SELECT address FROM etc_hosts LIMIT 1;' | osqueryi | tee /dev/console | grep -q '127.0.0.1'")
+
+      # osquery binaries respect configuration from the Nix config option.
+      machine.succeed("echo 'SELECT value FROM osquery_flags WHERE name = \"utc\";' | osqueryi | tee /dev/console | grep -q ${boolToString utc}")
+
+      # osquery binaries respect configuration from the Nix flags option.
+      machine.succeed("echo 'SELECT value FROM osquery_flags WHERE name = \"config_refresh\";' | osqueryi | tee /dev/console | grep -q ${config_refresh}")
+
+      # Demonstrate that osquery binaries prefer configuration plugin options over CLI flags.
+      # https://osquery.readthedocs.io/en/latest/deployment/configuration/#options.
+      machine.succeed("echo 'SELECT value FROM osquery_flags WHERE name = \"nullvalue\";' | osqueryi | tee /dev/console | grep -q ${nullvalue}")
+
+      # Module creates directories for default database_path and pidfile flag values.
+      machine.succeed("test -d $(dirname ${cfg.flags.database_path})")
+      machine.succeed("test -d $(dirname ${cfg.flags.pidfile})")
+    '';
+})

pkgs/tools/system/osquery/Remove-circular-definition-of-AUDIT_FILTER_EXCLUDE.patch

@@ -0,0 +1,25 @@
+From: Jack Baldry <jack.baldry@grafana.com>
+Date: Tue, 15 Nov 2022 15:40:31 -0400
+Subject: [PATCH] Remove circular definition of AUDIT_FILTER_EXCLUDE
+
+https://github.com/osquery/osquery/issues/6551
+
+Signed-off-by: Jack Baldry <jack.baldry@grafana.com>
+---
+ libraries/cmake/source/libaudit/src/lib/libaudit.h | 1 -
+ 1 file changed, 1 deletion(-)
+
+diff --git a/libraries/cmake/source/libaudit/src/lib/libaudit.h b/libraries/cmake/source/libaudit/src/libaudit.h
+--- a/libraries/cmake/source/libaudit/src/lib/libaudit.h
++++ b/libraries/cmake/source/libaudit/src/lib/libaudit.h
+@@ -260,7 +260,6 @@ extern "C" {
+ #define AUDIT_KEY_SEPARATOR 0x01
+ 
+ /* These are used in filter control */
+-#define AUDIT_FILTER_EXCLUDE	AUDIT_FILTER_TYPE
+ #define AUDIT_FILTER_MASK	0x07	/* Mask to get actual filter */
+ #define AUDIT_FILTER_UNSET	0x80	/* This value means filter is unset */
+ 
+-- 
+2.38.1
+

pkgs/tools/system/osquery/Remove-git-reset.patch

@@ -0,0 +1,37 @@
+From: Jack Baldry <jack.baldry@grafana.com>
+Date: Tue, 15 Nov 2022 13:48:07 -0400
+Subject: [PATCH] Remove git reset
+
+This is not required for nixpkgs builds because we are not working in
+the source repository and therefore do not need to be careful about
+updating submodule content.
+
+Signed-off-by: Jack Baldry <jack.baldry@grafana.com>
+---
+ libraries/cmake/source/modules/utils.cmake | 11 -----------
+ 1 file changed, 11 deletions(-)
+
+diff --git a/libraries/cmake/source/modules/utils.cmake b/libraries/cmake/source/modules/utils.cmake
+--- a/libraries/cmake/source/modules/utils.cmake
++++ b/libraries/cmake/source/modules/utils.cmake
+@@ -102,17 +102,6 @@ function(patchSubmoduleSourceCode library_name patches_dir source_dir apply_to_d
+     file(COPY "${source_dir}" DESTINATION "${parent_dir}")
+   endif()
+ 
+-  # We need to restore the source code to its original state, pre patch
+-  execute_process(
+-    COMMAND "${GIT_EXECUTABLE}" reset --hard HEAD
+-    RESULT_VARIABLE process_exit_code
+-    WORKING_DIRECTORY "${source_dir}"
+-  )
+-
+-  if(NOT ${process_exit_code} EQUAL 0)
+-    message(FATAL_ERROR "Failed to git reset the following submodule: \"${source_dir}\"")
+-  endif()
+-
+   set(patchSubmoduleSourceCode_Patched TRUE PARENT_SCOPE)
+ endfunction()
+ 
+-- 
+2.38.1
+

pkgs/tools/system/osquery/Remove-system-controls-table.patch

@@ -0,0 +1,157 @@
+From: Jack Baldry <jack.baldry@grafana.com>
+Date: Wed, 16 Nov 2022 22:00:06 -0400
+Subject: [PATCH] Remove system controls table
+
+Relies on <sys/sysctl.h> which is not present in glibc since 2.32.
+
+Signed-off-by: Jack Baldry <jack.baldry@grafana.com>
+---
+ osquery/tables/system/CMakeLists.txt         |  4 --
+ specs/CMakeLists.txt                         |  1 -
+ specs/posix/system_controls.table            | 21 -------
+ tests/integration/tables/system_controls.cpp | 61 --------------------
+ 4 files changed, 87 deletions(-)
+ delete mode 100644 specs/posix/system_controls.table
+ delete mode 100644 tests/integration/tables/system_controls.cpp
+
+diff --git a/osquery/tables/system/CMakeLists.txt b/osquery/tables/system/CMakeLists.txt
+--- a/osquery/tables/system/CMakeLists.txt
++++ b/osquery/tables/system/CMakeLists.txt
+@@ -43,7 +43,6 @@ function(generateOsqueryTablesSystemSystemtable)
+       posix/smbios_utils.cpp
+       posix/sudoers.cpp
+       posix/suid_bin.cpp
+-      posix/system_controls.cpp
+       posix/ulimit_info.cpp
+     )
+   endif()
+@@ -82,7 +81,6 @@ function(generateOsqueryTablesSystemSystemtable)
+       linux/shared_memory.cpp
+       linux/smbios_tables.cpp
+       linux/startup_items.cpp
+-      linux/sysctl_utils.cpp
+       linux/system_info.cpp
+       linux/usb_devices.cpp
+       linux/user_groups.cpp
+@@ -156,7 +154,6 @@ function(generateOsqueryTablesSystemSystemtable)
+       darwin/smbios_tables.cpp
+       darwin/smc_keys.cpp
+       darwin/startup_items.cpp
+-      darwin/sysctl_utils.cpp
+       darwin/system_extensions.mm
+       darwin/system_info.cpp
+       darwin/time_machine.cpp
+@@ -326,7 +323,6 @@ function(generateOsqueryTablesSystemSystemtable)
+       posix/shell_history.h
+       posix/ssh_keys.h
+       posix/sudoers.h
+-      posix/sysctl_utils.h
+       posix/last.h
+       posix/openssl_utils.h
+       posix/authorized_keys.h
+diff --git a/specs/CMakeLists.txt b/specs/CMakeLists.txt
+--- a/specs/CMakeLists.txt
++++ b/specs/CMakeLists.txt
+@@ -246,7 +246,6 @@ function(generateNativeTables)
+     "posix/socket_events.table:linux,macos"
+     "posix/sudoers.table:linux,macos,freebsd"
+     "posix/suid_bin.table:linux,macos,freebsd"
+-    "posix/system_controls.table:linux,macos,freebsd"
+     "posix/ulimit_info.table:linux,macos,freebsd"
+     "posix/usb_devices.table:linux,macos"
+     "posix/user_events.table:linux,macos,freebsd"
+diff --git a/specs/posix/system_controls.table b/specs/posix/system_controls.table
+deleted file mode 100644
+--- a/specs/posix/system_controls.table
++++ /dev/null
+@@ -1,21 +0,0 @@
+-table_name("system_controls")
+-description("sysctl names, values, and settings information.")
+-schema([
+-    Column("name", TEXT, "Full sysctl MIB name", index=True),
+-    Column("oid", TEXT, "Control MIB", additional=True),
+-    Column("subsystem", TEXT, "Subsystem ID, control type", additional=True),
+-    Column("current_value", TEXT, "Value of setting"),
+-    Column("config_value", TEXT, "The MIB value set in /etc/sysctl.conf"),
+-    Column("type", TEXT, "Data type"),
+-])
+-extended_schema(DARWIN, [
+-    Column("field_name", TEXT, "Specific attribute of opaque type"),
+-])
+-
+-implementation("system_controls@genSystemControls")
+-fuzz_paths([
+-    "/run/sysctl.d/",
+-    "/usr/lib/sysctl.d/",
+-    "/lib/sysctl.d/",
+-    "/sys"
+-])
+diff --git a/tests/integration/tables/system_controls.cpp b/tests/integration/tables/system_controls.cpp
+deleted file mode 100644
+--- a/tests/integration/tables/system_controls.cpp
++++ /dev/null
+@@ -1,61 +0,0 @@
+-/**
+- * Copyright (c) 2014-present, The osquery authors
+- *
+- * This source code is licensed as defined by the LICENSE file found in the
+- * root directory of this source tree.
+- *
+- * SPDX-License-Identifier: (Apache-2.0 OR GPL-2.0-only)
+- */
+-
+-// Sanity check integration test for system_controls
+-// Spec file: specs/posix/system_controls.table
+-
+-#include <osquery/tests/integration/tables/helper.h>
+-
+-namespace osquery {
+-namespace table_tests {
+-namespace {
+-
+-class SystemControlsTest : public testing::Test {
+- protected:
+-  void SetUp() override {
+-    setUpEnvironment();
+-  }
+-};
+-
+-TEST_F(SystemControlsTest, test_sanity) {
+-  auto const rows = execute_query("select * from system_controls");
+-  auto const row_map = ValidationMap{
+-      {"name", NonEmptyString},
+-      {"oid", NormalType},
+-      {"subsystem",
+-       SpecificValuesCheck{"",
+-                           "abi",
+-                           "debug",
+-                           "dev",
+-                           "fs",
+-                           "fscache",
+-                           "hw",
+-                           "kern",
+-                           "kernel",
+-                           "machdep",
+-                           "net",
+-                           "sunrpc",
+-                           "user",
+-                           "vfs",
+-                           "vm"}},
+-      {"current_value", NormalType},
+-      {"config_value", NormalType},
+-      {"type",
+-       SpecificValuesCheck{
+-           "", "node", "int", "string", "quad", "opaque", "struct"}},
+-#ifdef __APPLE__
+-      {"field_name", NormalType},
+-#endif
+-  };
+-  validate_rows(rows, row_map);
+-}
+-
+-} // namespace
+-} // namespace table_tests
+-} // namespace osquery
+-- 
+2.38.1
+

pkgs/tools/system/osquery/Use-locale.h-instead-of-removed-xlocale.h-header.patch

@@ -0,0 +1,29 @@
+From: Jack Baldry <jack.baldry@grafana.com>
+Date: Tue, 15 Nov 2022 14:34:33 -0400
+Subject: [PATCH] Use locale.h instead of removed xlocale.h header
+
+https://sourceware.org/glibc/wiki/Release/2.26#Removal_of_.27xlocale.h.27
+
+Signed-off-by: Jack Baldry <jack.baldry@grafana.com>
+---
+ libraries/cmake/source/augeas/gnulib/generated/linux/x86_64/lib/locale.h  | 4 ++--
+ 1 file changed, 2 insertions(+), 2 deletions(-)
+
+diff --git a/libraries/cmake/source/augeas/gnulib/generated/linux/x86_64/lib/locale.h b/libraries/cmake/source/augeas/gnulib/generated/linux/x86_64/lib/locale.h
+--- a/libraries/cmake/source/augeas/gnulib/generated/linux/x86_64/lib/locale.h
++++ b/libraries/cmake/source/augeas/gnulib/generated/linux/x86_64/lib/locale.h
+@@ -48,9 +48,9 @@
+ /* NetBSD 5.0 mis-defines NULL.  */
+ #include <stddef.h>
+ 
+-/* Mac OS X 10.5 defines the locale_t type in <xlocale.h>.  */
++/* Mac OS X 10.5 defines the locale_t type in <locale.h>.  */
+ #if 1
+-# include <xlocale.h>
++# include <locale.h>
+ #endif
+ 
+ /* The definitions of _GL_FUNCDECL_RPL etc. are copied here.  */
+-- 
+2.38.1
+

pkgs/tools/system/osquery/default.nix

@@ -0,0 +1,85 @@
+{ lib
+, cmake
+, fetchFromGitHub
+, git
+, llvmPackages
+, nixosTests
+, overrideCC
+, perl
+, python3
+, stdenv
+, openssl_1_1
+}:
+
+let
+  buildStdenv = overrideCC stdenv llvmPackages.clangUseLLVM;
+in
+buildStdenv.mkDerivation rec {
+  pname = "osquery";
+  version = "5.5.1";
+
+  src = fetchFromGitHub {
+    owner = "osquery";
+    repo = "osquery";
+    rev = version;
+    fetchSubmodules = true;
+    sha256 = "sha256-Q6PQVnBjAjAlR725fyny+RhQFUNwxWGjLDuS5p9JKlU=";
+  };
+
+  patches = [
+    ./Remove-git-reset.patch
+    ./Use-locale.h-instead-of-removed-xlocale.h-header.patch
+    ./Remove-circular-definition-of-AUDIT_FILTER_EXCLUDE.patch
+    # For current state of compilation against glibc in the clangWithLLVM toolchain, refer to the upstream issue in https://github.com/osquery/osquery/issues/7823.
+    ./Remove-system-controls-table.patch
+  ];
+
+
+  buildInputs = [
+    llvmPackages.libunwind
+  ];
+  nativeBuildInputs = [
+    cmake
+    git
+    perl
+    python3
+  ];
+
+  postPatch = ''
+    substituteInPlace cmake/install_directives.cmake --replace "/control" "control"
+    # This is required to build libarchive with our glibc version
+    # which provides the ARC4RANDOM_BUF function
+    substituteInPlace libraries/cmake/source/libarchive/CMakeLists.txt --replace "  target_compile_definitions(thirdparty_libarchive PRIVATE" "  target_compile_definitions(thirdparty_libarchive PRIVATE HAVE_ARC4RANDOM_BUF"
+    # We need to override this hash because we use our own openssl 1.1 version
+    substituteInPlace libraries/cmake/formula/openssl/CMakeLists.txt --replace "d7939ce614029cdff0b6c20f0e2e5703158a489a72b2507b8bd51bf8c8fd10ca" "e2f8d84b523eecd06c7be7626830370300fbcc15386bf5142d72758f6963ebc6"
+    cat libraries/cmake/formula/openssl/CMakeLists.txt
+  '';
+
+  # For explanation of these deletions, refer to the ./Use-locale.h-instead-of-removed-xlocale.h-header.patch file.
+  preConfigure = ''
+    find libraries/cmake/source -name 'config.h' -exec sed -i '/#define HAVE_XLOCALE_H 1/d' {} \;
+  '';
+
+  cmakeFlags = [
+    "-DOSQUERY_VERSION=${version}"
+    "-DOSQUERY_OPENSSL_ARCHIVE_PATH=${openssl_1_1.src}"
+  ];
+
+  postFixup = ''
+    patchelf --set-rpath "${llvmPackages.libunwind}/lib:$(patchelf --print-rpath $out/bin/osqueryd)" "$out/bin/osqueryd"
+  '';
+
+  passthru.tests.osquery = nixosTests.osquery;
+
+  meta = with lib; {
+    description = "SQL powered operating system instrumentation, monitoring, and analytics.";
+    longDescription = ''
+      The system controls table is not included as it does not presently compile with glibc >= 2.32.
+      For more information, refer to https://github.com/osquery/osquery/issues/7823
+    '';
+    homepage = "https://osquery.io";
+    license = licenses.bsd3;
+    platforms = platforms.linux;
+    maintainers = with maintainers; [ znewman01 lewo ];
+  };
+}

pkgs/top-level/all-packages.nix

@@ -1789,6 +1789,8 @@ with pkgs;
 
   openbugs = pkgsi686Linux.callPackage ../applications/science/machine-learning/openbugs { };
 
+  osquery = callPackage ../tools/system/osquery { };
+
   paperview = callPackage ../tools/X11/paperview { };
 
   pferd = callPackage ../tools/misc/pferd { };