From ebf346ff0416630c21166bb8568812075ae853bf Mon Sep 17 00:00:00 2001 From: Jade Lovelace Date: Fri, 8 Mar 2024 12:31:11 -0800 Subject: [PATCH] buildbot: tie the knot through a scope to make it overridable Currently it is impossible to overlay buildbot since all the references between components are directly bound to the values in scope. Let's fix this by introducing a scope so you can overrideScope parts of buildbot. I also changed buildbot-worker to use our overridden python, which is no functional change, but makes things more consistent. --- .../buildbot/default.nix | 41 ++-- .../buildbot/master.nix | 202 +++++++++--------- pkgs/top-level/all-packages.nix | 4 +- 3 files changed, 121 insertions(+), 126 deletions(-) diff --git a/pkgs/development/tools/continuous-integration/buildbot/default.nix b/pkgs/development/tools/continuous-integration/buildbot/default.nix index 019ba4579d4a..b2ec827f7526 100644 --- a/pkgs/development/tools/continuous-integration/buildbot/default.nix +++ b/pkgs/development/tools/continuous-integration/buildbot/default.nix @@ -1,36 +1,31 @@ -{ python3 -, fetchPypi +{ lib +, newScope +, python3 , recurseIntoAttrs -, callPackage }: -let +# Take packages from self first, then python.pkgs (and secondarily pkgs) +lib.makeScope (self: newScope (self.python.pkgs // self)) (self: { python = python3.override { packageOverrides = self: super: { sqlalchemy = super.sqlalchemy_1_4; - moto = super.moto.overridePythonAttrs (oldAttrs: rec { + moto = super.moto.overridePythonAttrs (oldAttrs: { # a lot of tests -> very slow, we already build them when building python packages doCheck = false; }); }; }; - buildbot-pkg = python.pkgs.callPackage ./pkg.nix { - inherit buildbot; - }; - buildbot-worker = python3.pkgs.callPackage ./worker.nix { - inherit buildbot; - }; - buildbot = python.pkgs.callPackage ./master.nix { - inherit buildbot-pkg buildbot-worker buildbot-plugins; - }; - buildbot-plugins = recurseIntoAttrs (callPackage ./plugins.nix { - inherit buildbot-pkg; - }); -in -{ - inherit buildbot buildbot-plugins buildbot-worker; - buildbot-ui = buildbot.withPlugins (with buildbot-plugins; [ www ]); - buildbot-full = buildbot.withPlugins (with buildbot-plugins; [ + buildbot-pkg = self.callPackage ./pkg.nix { }; + + buildbot-worker = self.callPackage ./worker.nix { }; + + buildbot = self.callPackage ./master.nix { }; + + buildbot-plugins = recurseIntoAttrs (self.callPackage ./plugins.nix { }); + + buildbot-ui = self.buildbot.withPlugins (with self.buildbot-plugins; [ www ]); + + buildbot-full = self.buildbot.withPlugins (with self.buildbot-plugins; [ www console-view waterfall-view grid-view wsgi-dashboards badges ]); -} +}) diff --git a/pkgs/development/tools/continuous-integration/buildbot/master.nix b/pkgs/development/tools/continuous-integration/buildbot/master.nix index 98626ed36bf4..68566ca93c79 100644 --- a/pkgs/development/tools/continuous-integration/buildbot/master.nix +++ b/pkgs/development/tools/continuous-integration/buildbot/master.nix @@ -1,9 +1,11 @@ { lib , stdenv -, buildPythonPackage , buildPythonApplication , fetchPypi , makeWrapper +# Tie withPlugins through the fixed point here, so it will receive an +# overridden version properly +, buildbot , pythonOlder , python , twisted @@ -38,13 +40,12 @@ , unidiff , glibcLocales , nixosTests -, callPackage }: let withPlugins = plugins: buildPythonApplication { - pname = "${package.pname}-with-plugins"; - inherit (package) version; + pname = "${buildbot.pname}-with-plugins"; + inherit (buildbot) version; format = "other"; dontUnpack = true; @@ -55,110 +56,109 @@ let makeWrapper ]; - propagatedBuildInputs = plugins ++ package.propagatedBuildInputs; + propagatedBuildInputs = plugins ++ buildbot.propagatedBuildInputs; installPhase = '' - makeWrapper ${package}/bin/buildbot $out/bin/buildbot \ - --prefix PYTHONPATH : "${package}/${python.sitePackages}:$PYTHONPATH" - ln -sfv ${package}/lib $out/lib + makeWrapper ${buildbot}/bin/buildbot $out/bin/buildbot \ + --prefix PYTHONPATH : "${buildbot}/${python.sitePackages}:$PYTHONPATH" + ln -sfv ${buildbot}/lib $out/lib ''; - passthru = package.passthru // { + passthru = buildbot.passthru // { withPlugins = morePlugins: withPlugins (morePlugins ++ plugins); }; }; +in +buildPythonApplication rec { + pname = "buildbot"; + version = "3.11.1"; + format = "pyproject"; - package = buildPythonApplication rec { - pname = "buildbot"; - version = "3.11.1"; - format = "pyproject"; + disabled = pythonOlder "3.8"; - disabled = pythonOlder "3.8"; - - src = fetchPypi { - inherit pname version; - hash = "sha256-ruYW1sVoGvFMi+NS+xiNsn0Iq2RmKlax4bxHgYrj6ZY="; - }; - - propagatedBuildInputs = [ - # core - twisted - jinja2 - msgpack - zope-interface - sqlalchemy - alembic - python-dateutil - txaio - autobahn - pyjwt - pyyaml - setuptools - croniter - importlib-resources - packaging - unidiff - ] - # tls - ++ twisted.optional-dependencies.tls; - - nativeCheckInputs = [ - treq - txrequests - pypugjs - boto3 - moto - markdown - lz4 - setuptools-trial - buildbot-worker - buildbot-pkg - buildbot-plugins.www - parameterized - git - openssh - glibcLocales - ]; - - patches = [ - # This patch disables the test that tries to read /etc/os-release which - # is not accessible in sandboxed builds. - ./skip_test_linux_distro.patch - ]; - - postPatch = '' - substituteInPlace buildbot/scripts/logwatcher.py --replace '/usr/bin/tail' "$(type -P tail)" - ''; - - # Silence the depreciation warning from SqlAlchemy - SQLALCHEMY_SILENCE_UBER_WARNING = 1; - - # TimeoutErrors on slow machines -> aarch64 - doCheck = !stdenv.isAarch64; - - preCheck = '' - export LC_ALL="en_US.UTF-8" - export PATH="$out/bin:$PATH" - - # remove testfile which is missing configuration file from sdist - rm buildbot/test/integration/test_graphql.py - # tests in this file are flaky, see https://github.com/buildbot/buildbot/issues/6776 - rm buildbot/test/integration/test_try_client.py - ''; - - passthru = { - inherit withPlugins; - tests.buildbot = nixosTests.buildbot; - updateScript = ./update.sh; - }; - - meta = with lib; { - description = "An open-source continuous integration framework for automating software build, test, and release processes"; - homepage = "https://buildbot.net/"; - changelog = "https://github.com/buildbot/buildbot/releases/tag/v${version}"; - maintainers = teams.buildbot.members; - license = licenses.gpl2Only; - broken = stdenv.isDarwin; - }; + src = fetchPypi { + inherit pname version; + hash = "sha256-ruYW1sVoGvFMi+NS+xiNsn0Iq2RmKlax4bxHgYrj6ZY="; }; -in package + + propagatedBuildInputs = [ + # core + twisted + jinja2 + msgpack + zope-interface + sqlalchemy + alembic + python-dateutil + txaio + autobahn + pyjwt + pyyaml + setuptools + croniter + importlib-resources + packaging + unidiff + ] + # tls + ++ twisted.optional-dependencies.tls; + + nativeCheckInputs = [ + treq + txrequests + pypugjs + boto3 + moto + markdown + lz4 + setuptools-trial + buildbot-worker + buildbot-pkg + buildbot-plugins.www + parameterized + git + openssh + glibcLocales + ]; + + patches = [ + # This patch disables the test that tries to read /etc/os-release which + # is not accessible in sandboxed builds. + ./skip_test_linux_distro.patch + ]; + + postPatch = '' + substituteInPlace buildbot/scripts/logwatcher.py --replace '/usr/bin/tail' "$(type -P tail)" + ''; + + # Silence the depreciation warning from SqlAlchemy + SQLALCHEMY_SILENCE_UBER_WARNING = 1; + + # TimeoutErrors on slow machines -> aarch64 + doCheck = !stdenv.isAarch64; + + preCheck = '' + export LC_ALL="en_US.UTF-8" + export PATH="$out/bin:$PATH" + + # remove testfile which is missing configuration file from sdist + rm buildbot/test/integration/test_graphql.py + # tests in this file are flaky, see https://github.com/buildbot/buildbot/issues/6776 + rm buildbot/test/integration/test_try_client.py + ''; + + passthru = { + inherit withPlugins; + tests.buildbot = nixosTests.buildbot; + updateScript = ./update.sh; + }; + + meta = with lib; { + description = "An open-source continuous integration framework for automating software build, test, and release processes"; + homepage = "https://buildbot.net/"; + changelog = "https://github.com/buildbot/buildbot/releases/tag/v${version}"; + maintainers = teams.buildbot.members; + license = licenses.gpl2Only; + broken = stdenv.isDarwin; + }; +} diff --git a/pkgs/top-level/all-packages.nix b/pkgs/top-level/all-packages.nix index c10847fd3188..237f14adbc4f 100644 --- a/pkgs/top-level/all-packages.nix +++ b/pkgs/top-level/all-packages.nix @@ -3581,8 +3581,8 @@ with pkgs; bucklespring-libinput = callPackage ../applications/audio/bucklespring { }; bucklespring-x11 = callPackage ../applications/audio/bucklespring { legacy = true; }; - inherit (python3.pkgs.callPackage ../development/tools/continuous-integration/buildbot {}) - buildbot buildbot-ui buildbot-full buildbot-plugins buildbot-worker; + buildbotPackages = recurseIntoAttrs (python3.pkgs.callPackage ../development/tools/continuous-integration/buildbot { }); + inherit (buildbotPackages) buildbot buildbot-ui buildbot-full buildbot-plugins buildbot-worker; bunyan-rs = callPackage ../development/tools/bunyan-rs { };