From 90b750456ea62408dc4c0bbe68adabdd96e381a7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Fri, 7 Apr 2023 18:34:08 +0200 Subject: [PATCH] buildbot: move out of python3.pkgs By moving it out we can start apply overrides to python dependendencies starting with sqlalchemy. This fixes the build with the current version. --- .../buildbot/master.nix | 11 ++--- .../buildbot/worker.nix | 7 ++-- nixos/tests/buildbot.nix | 4 +- .../buildbot/default.nix | 42 +++++++++++++++++++ .../buildbot/master.nix} | 8 ++-- .../continuous-integration}/buildbot/pkg.nix | 0 .../buildbot/plugins.nix | 3 +- .../buildbot/skip_test_linux_distro.patch | 0 .../buildbot/update.sh | 0 .../buildbot/worker.nix | 0 pkgs/top-level/all-packages.nix | 6 +-- pkgs/top-level/python-aliases.nix | 6 +++ pkgs/top-level/python-packages.nix | 12 ------ 13 files changed, 68 insertions(+), 31 deletions(-) create mode 100644 pkgs/development/tools/continuous-integration/buildbot/default.nix rename pkgs/development/{python-modules/buildbot/default.nix => tools/continuous-integration/buildbot/master.nix} (96%) rename pkgs/development/{python-modules => tools/continuous-integration}/buildbot/pkg.nix (100%) rename pkgs/development/{python-modules => tools/continuous-integration}/buildbot/plugins.nix (97%) rename pkgs/development/{python-modules => tools/continuous-integration}/buildbot/skip_test_linux_distro.patch (100%) rename pkgs/development/{python-modules => tools/continuous-integration}/buildbot/update.sh (100%) rename pkgs/development/{python-modules => tools/continuous-integration}/buildbot/worker.nix (100%) diff --git a/nixos/modules/services/continuous-integration/buildbot/master.nix b/nixos/modules/services/continuous-integration/buildbot/master.nix index 5666199c4845..595374ea1e5b 100644 --- a/nixos/modules/services/continuous-integration/buildbot/master.nix +++ b/nixos/modules/services/continuous-integration/buildbot/master.nix @@ -8,7 +8,8 @@ let cfg = config.services.buildbot-master; opt = options.services.buildbot-master; - python = cfg.package.pythonModule; + package = pkgs.python3.pkgs.toPythonModule cfg.package; + python = package.pythonModule; escapeStr = escape [ "'" ]; @@ -212,10 +213,10 @@ in { package = mkOption { type = types.package; - default = pkgs.python3Packages.buildbot-full; - defaultText = literalExpression "pkgs.python3Packages.buildbot-full"; + default = pkgs.buildbot-full; + defaultText = literalExpression "pkgs.buildbot-full"; description = lib.mdDoc "Package to use for buildbot."; - example = literalExpression "pkgs.python3Packages.buildbot"; + example = literalExpression "pkgs.buildbot"; }; packages = mkOption { @@ -255,7 +256,7 @@ in { after = [ "network-online.target" ]; wantedBy = [ "multi-user.target" ]; path = cfg.packages ++ cfg.pythonPackages python.pkgs; - environment.PYTHONPATH = "${python.withPackages (self: cfg.pythonPackages self ++ [ cfg.package ])}/${python.sitePackages}"; + environment.PYTHONPATH = "${python.withPackages (self: cfg.pythonPackages self ++ [ package ])}/${python.sitePackages}"; preStart = '' mkdir -vp "${cfg.buildbotDir}" diff --git a/nixos/modules/services/continuous-integration/buildbot/worker.nix b/nixos/modules/services/continuous-integration/buildbot/worker.nix index 52c41c4a7584..7e78b8935f81 100644 --- a/nixos/modules/services/continuous-integration/buildbot/worker.nix +++ b/nixos/modules/services/continuous-integration/buildbot/worker.nix @@ -8,7 +8,8 @@ let cfg = config.services.buildbot-worker; opt = options.services.buildbot-worker; - python = cfg.package.pythonModule; + package = pkgs.python3.pkgs.toPythonModule cfg.package; + python = package.pythonModule; tacFile = pkgs.writeText "aur-buildbot-worker.tac" '' import os @@ -129,7 +130,7 @@ in { package = mkOption { type = types.package; - default = pkgs.python3Packages.buildbot-worker; + default = pkgs.buildbot-worker; defaultText = literalExpression "pkgs.python3Packages.buildbot-worker"; description = lib.mdDoc "Package to use for buildbot worker."; example = literalExpression "pkgs.python2Packages.buildbot-worker"; @@ -168,7 +169,7 @@ in { after = [ "network.target" "buildbot-master.service" ]; wantedBy = [ "multi-user.target" ]; path = cfg.packages; - environment.PYTHONPATH = "${python.withPackages (p: [ cfg.package ])}/${python.sitePackages}"; + environment.PYTHONPATH = "${python.withPackages (p: [ package ])}/${python.sitePackages}"; preStart = '' mkdir -vp "${cfg.buildbotDir}/info" diff --git a/nixos/tests/buildbot.nix b/nixos/tests/buildbot.nix index 977c728835f0..467c8d8baff4 100644 --- a/nixos/tests/buildbot.nix +++ b/nixos/tests/buildbot.nix @@ -23,7 +23,7 @@ import ./make-test-python.nix { ]; }; networking.firewall.allowedTCPPorts = [ 8010 8011 9989 ]; - environment.systemPackages = with pkgs; [ git python3Packages.buildbot-full ]; + environment.systemPackages = with pkgs; [ git buildbot-full ]; }; bbworker = { pkgs, ... }: { @@ -31,7 +31,7 @@ import ./make-test-python.nix { enable = true; masterUrl = "bbmaster:9989"; }; - environment.systemPackages = with pkgs; [ git python3Packages.buildbot-worker ]; + environment.systemPackages = with pkgs; [ git buildbot-worker ]; }; gitrepo = { pkgs, ... }: { diff --git a/pkgs/development/tools/continuous-integration/buildbot/default.nix b/pkgs/development/tools/continuous-integration/buildbot/default.nix new file mode 100644 index 000000000000..ec3a4a81f60f --- /dev/null +++ b/pkgs/development/tools/continuous-integration/buildbot/default.nix @@ -0,0 +1,42 @@ +{ python3 +, recurseIntoAttrs +, callPackage +}: +let + python = python3.override { + packageOverrides = self: super: { + sqlalchemy = super.sqlalchemy.overridePythonAttrs (oldAttrs: rec { + version = "1.4.40"; + src = super.fetchPypi { + pname = "SQLAlchemy"; + inherit version; + hash = "sha256-RKZgUGCAzJdeHfpXdv5fYxXdxiane1C/Du4YsDieomU="; + }; + }); + moto = super.moto.overridePythonAttrs (oldAttrs: rec { + # a lot of tests -> very slow, we already build them when building python packages + doCheck = false; + }); + }; + }; + + buildbot-pkg = python.pkgs.callPackage ./pkg.nix { + inherit buildbot; + }; + buildbot-worker = python3.pkgs.callPackage ./worker.nix { + inherit buildbot; + }; + buildbot = python.pkgs.callPackage ./master.nix { + inherit buildbot-pkg buildbot-worker buildbot-plugins; + }; + buildbot-plugins = recurseIntoAttrs (callPackage ./plugins.nix { + inherit buildbot-pkg; + }); +in +{ + inherit buildbot buildbot-plugins buildbot-worker; + buildbot-ui = buildbot.withPlugins (with buildbot-plugins; [ www ]); + buildbot-full = buildbot.withPlugins (with buildbot-plugins; [ + www console-view waterfall-view grid-view wsgi-dashboards badges + ]); +} diff --git a/pkgs/development/python-modules/buildbot/default.nix b/pkgs/development/tools/continuous-integration/buildbot/master.nix similarity index 96% rename from pkgs/development/python-modules/buildbot/default.nix rename to pkgs/development/tools/continuous-integration/buildbot/master.nix index c1a0de718200..6e59e31867b6 100644 --- a/pkgs/development/python-modules/buildbot/default.nix +++ b/pkgs/development/tools/continuous-integration/buildbot/master.nix @@ -1,6 +1,7 @@ { lib , stdenv , buildPythonPackage +, buildPythonApplication , fetchPypi , makeWrapper , pythonOlder @@ -25,17 +26,18 @@ , lz4 , setuptoolsTrial , buildbot-worker -, buildbot-pkg , buildbot-plugins +, buildbot-pkg , parameterized , git , openssh , glibcLocales , nixosTests +, callPackage }: let - withPlugins = plugins: buildPythonPackage { + withPlugins = plugins: buildPythonApplication { pname = "${package.pname}-with-plugins"; inherit (package) version; format = "other"; @@ -61,7 +63,7 @@ let }; }; - package = buildPythonPackage rec { + package = buildPythonApplication rec { pname = "buildbot"; version = "3.7.0"; format = "setuptools"; diff --git a/pkgs/development/python-modules/buildbot/pkg.nix b/pkgs/development/tools/continuous-integration/buildbot/pkg.nix similarity index 100% rename from pkgs/development/python-modules/buildbot/pkg.nix rename to pkgs/development/tools/continuous-integration/buildbot/pkg.nix diff --git a/pkgs/development/python-modules/buildbot/plugins.nix b/pkgs/development/tools/continuous-integration/buildbot/plugins.nix similarity index 97% rename from pkgs/development/python-modules/buildbot/plugins.nix rename to pkgs/development/tools/continuous-integration/buildbot/plugins.nix index df9efd66efe8..42a77fa0f4bc 100644 --- a/pkgs/development/python-modules/buildbot/plugins.nix +++ b/pkgs/development/tools/continuous-integration/buildbot/plugins.nix @@ -1,5 +1,4 @@ -{ lib, buildPythonPackage, fetchPypi, buildbot-pkg, mock, cairosvg, klein, jinja2 }: - +{ lib, buildPythonPackage, fetchPypi, callPackage, mock, cairosvg, klein, jinja2, buildbot-pkg }: { www = buildPythonPackage rec { pname = "buildbot-www"; diff --git a/pkgs/development/python-modules/buildbot/skip_test_linux_distro.patch b/pkgs/development/tools/continuous-integration/buildbot/skip_test_linux_distro.patch similarity index 100% rename from pkgs/development/python-modules/buildbot/skip_test_linux_distro.patch rename to pkgs/development/tools/continuous-integration/buildbot/skip_test_linux_distro.patch diff --git a/pkgs/development/python-modules/buildbot/update.sh b/pkgs/development/tools/continuous-integration/buildbot/update.sh similarity index 100% rename from pkgs/development/python-modules/buildbot/update.sh rename to pkgs/development/tools/continuous-integration/buildbot/update.sh diff --git a/pkgs/development/python-modules/buildbot/worker.nix b/pkgs/development/tools/continuous-integration/buildbot/worker.nix similarity index 100% rename from pkgs/development/python-modules/buildbot/worker.nix rename to pkgs/development/tools/continuous-integration/buildbot/worker.nix diff --git a/pkgs/top-level/all-packages.nix b/pkgs/top-level/all-packages.nix index 01b088004bbf..82929439254f 100644 --- a/pkgs/top-level/all-packages.nix +++ b/pkgs/top-level/all-packages.nix @@ -3079,10 +3079,8 @@ with pkgs; bucklespring-libinput = callPackage ../applications/audio/bucklespring { }; bucklespring-x11 = callPackage ../applications/audio/bucklespring { legacy = true; }; - buildbot = with python3Packages; toPythonApplication buildbot; - buildbot-ui = with python3Packages; toPythonApplication buildbot-ui; - buildbot-full = with python3Packages; toPythonApplication buildbot-full; - buildbot-worker = with python3Packages; toPythonApplication buildbot-worker; + inherit (python3.pkgs.callPackage ../development/tools/continuous-integration/buildbot {}) + buildbot buildbot-ui buildbot-full buildbot-plugins buildbot-worker; bunyan-rs = callPackage ../development/tools/bunyan-rs { }; diff --git a/pkgs/top-level/python-aliases.nix b/pkgs/top-level/python-aliases.nix index 719fb88362ab..e39a0259b2ae 100644 --- a/pkgs/top-level/python-aliases.nix +++ b/pkgs/top-level/python-aliases.nix @@ -48,6 +48,12 @@ mapAliases ({ blockdiagcontrib-cisco = throw "blockdiagcontrib-cisco is not compatible with blockdiag 2.0.0 and has been removed."; # added 2020-11-29 bsblan = python-bsblan; # added 2022-11-04 btchip = btchip-python; # added 2023-03-03 + buildbot = throw "use pkgs.buildbot instead"; # added 2022-04-07 + buildbot-ui = throw "use pkgs.buildbot-ui instead"; # added 2022-04-07 + buildbot-full = throw "use pkgs.buildbot-full instead"; # added 2022-04-07 + buildbot-plugins = throw "use pkgs.buildbot-plugins instead"; # added 2022-04-07 + buildbot-worker = throw "use pkgs.buildbot-worker instead"; # added 2022-04-07 + buildbot-pkg = throw "buildbot-pkg has been removed, it's only internally used in buildbot"; # added 2022-04-07 bt_proximity = bt-proximity; # added 2021-07-02 BTrees = btrees; # added 2023-02-19 carrot = throw "carrot has been removed, as its development was discontinued in 2012"; # added 2022-01-18 diff --git a/pkgs/top-level/python-packages.nix b/pkgs/top-level/python-packages.nix index 627b11b5ad99..d7c214555f22 100644 --- a/pkgs/top-level/python-packages.nix +++ b/pkgs/top-level/python-packages.nix @@ -1513,18 +1513,6 @@ self: super: with self; { buienradar = callPackage ../development/python-modules/buienradar { }; - buildbot = callPackage ../development/python-modules/buildbot { }; - - buildbot-ui = self.buildbot.withPlugins (with self.buildbot-plugins; [ www ]); - - buildbot-full = self.buildbot.withPlugins (with self.buildbot-plugins; [ www console-view waterfall-view grid-view wsgi-dashboards badges ]); - - buildbot-pkg = callPackage ../development/python-modules/buildbot/pkg.nix { }; - - buildbot-plugins = pkgs.recurseIntoAttrs (callPackage ../development/python-modules/buildbot/plugins.nix { }); - - buildbot-worker = callPackage ../development/python-modules/buildbot/worker.nix { }; - build = callPackage ../development/python-modules/build { }; buildcatrust = callPackage ../development/python-modules/buildcatrust { };