pythonPackages: ensure all derivations provide python modules

This adds a test to ensure no new uses of `buildPythonApplication` can
be added to `python-packages.nix`.

Python packages can be grouped into two groups: 1) applications and 2)
packages providing importable modules. In `python-packages.nix` we only
want to have 2). 1) should be in the top-level package set.

To achieve this, all setup hooks need to be marked as being a setup hook.
For the setup hooks in the Python packages set this is done by creating
a new builder, `makePythonHook`.

Because there were issues with splicing, the file importing all the hooks
is converted to an extension. All non-packages were moved out of `python-packages.nix`
into `python-packages-base.nix`. The `keep` argument to `makeScopeWithSplicing
was cleaned up as well; there is no need to keep this one manually in sync
reducing the risk of breaking cross-compilation.

pkgs/development/interpreters/python/default.nix

@@ -31,13 +31,19 @@
     , pythonAttr ? null
     , self # is pythonOnHostForTarget
     }: let
-      pythonPackages = callPackage
+      pythonPackages = let
+        ensurePythonModules = items: let
+          providesSetupHook = lib.attrByPath [ "provides" "setupHook"] false;
+          notValid = value: (lib.isDerivation value) && !((pythonPackages.hasPythonModule value) || (providesSetupHook value));
+          func = name: value: if !(notValid value) then value else throw "${name} should use `buildPythonPackage` or `toPythonModule` if it is to be part of the Python packages set.";
+        in lib.mapAttrs func items;
+      in ensurePythonModules (callPackage
         # Function that when called
         # - imports python-packages.nix
         # - adds spliced package sets to the package set
         # - applies overrides from `packageOverrides` and `pythonPackagesOverlays`.
         ({ pkgs, stdenv, python, overrides }: let
-          pythonPackagesFun = import ../../../top-level/python-packages.nix {
+          pythonPackagesFun = import ./python-packages-base.nix {
             inherit stdenv pkgs lib;
             python = self;
           };
@@ -48,47 +54,19 @@
             selfHostHost = pythonOnHostForHost.pkgs;
             selfTargetTarget = pythonOnTargetForTarget.pkgs or {}; # There is no Python TargetTarget.
           };
-          keep = self: {
-            # TODO maybe only define these here so nothing is needed to be kept in sync.
-            inherit (self)
-              isPy27 isPy35 isPy36 isPy37 isPy38 isPy39 isPy310 isPy3k isPyPy pythonAtLeast pythonOlder
-              python bootstrapped-pip buildPythonPackage buildPythonApplication
-              fetchPypi
-              hasPythonModule requiredPythonModules makePythonPath disabledIf
-              toPythonModule toPythonApplication
-              buildSetupcfg
-
-              condaInstallHook
-              condaUnpackHook
-              eggUnpackHook
-              eggBuildHook
-              eggInstallHook
-              flitBuildHook
-              pipBuildHook
-              pipInstallHook
-              pytestCheckHook
-              pythonCatchConflictsHook
-              pythonImportsCheckHook
-              pythonNamespacesHook
-              pythonRecompileBytecodeHook
-              pythonRemoveBinBytecodeHook
-              pythonRemoveTestsDirHook
-              setuptoolsBuildHook
-              setuptoolsCheckHook
-              venvShellHook
-              wheelUnpackHook
-
-              wrapPython
-
-              pythonPackages
-
-              recursivePthLoader
-            ;
-          };
+          hooks = import ./hooks/default.nix;
+          keep = lib.extends hooks pythonPackagesFun;
           extra = _: {};
           optionalExtensions = cond: as: if cond then as else [];
+          pythonExtension = import ../../../top-level/python-packages.nix;
           python2Extension = import ../../../top-level/python2-packages.nix;
-          extensions = lib.composeManyExtensions ((optionalExtensions (!self.isPy3k) [python2Extension]) ++ pythonPackagesExtensions ++ [ overrides ]);
+          extensions = lib.composeManyExtensions ([
+            pythonExtension
+          ] ++ (optionalExtensions (!self.isPy3k) [
+            python2Extension
+          ]) ++ pythonPackagesExtensions ++ [
+            overrides
+          ]);
           aliases = self: super: lib.optionalAttrs config.allowAliases (import ../../../top-level/python-aliases.nix lib self super);
         in lib.makeScopeWithSplicing
           splicePackages
@@ -96,11 +74,11 @@
           otherSplices
           keep
           extra
-          (lib.extends (lib.composeExtensions aliases extensions) pythonPackagesFun))
+          (lib.extends (lib.composeExtensions aliases extensions) keep))
         {
           overrides = packageOverrides;
           python = self;
-        };
+        });
     in rec {
         isPy27 = pythonVersion == "2.7";
         isPy35 = pythonVersion == "3.5";