pulumiPackages: move implementation to pkgs/by-name (#386789)

Moves pulumiPackages to pkgs/by-name/pu/pulumi/plugins.
    Updates pulumiPackages implementation to use makeScopeWithSplicing.
    Renames pulumi-{go,nodejs,python} for consistency with other plugins (with an alias if config.allowAliases option is enabled in Nixpkgs).
    Adds testResourceSchema and pulumiTestHook for plugin version bumps in the next PR.

pkgs/by-name/pu/pulumi/extra/mk-pulumi-package.nix

@@ -0,0 +1,187 @@
+{
+  buildGoModule,
+  fetchFromGitHub,
+  python3Packages,
+}:
+let
+  mkBasePackage =
+    {
+      pname,
+      src,
+      version,
+      vendorHash,
+      cmd,
+      extraLdflags,
+      env,
+      ...
+    }@args:
+    buildGoModule (
+      rec {
+        inherit
+          pname
+          src
+          vendorHash
+          version
+          env
+          ;
+
+        sourceRoot = "${src.name}/provider";
+
+        subPackages = [ "cmd/${cmd}" ];
+
+        doCheck = false;
+
+        ldflags = [
+          "-s"
+          "-w"
+        ] ++ extraLdflags;
+      }
+      // args
+    );
+
+  mkPythonPackage =
+    {
+      meta,
+      pname,
+      src,
+      version,
+      ...
+    }:
+    python3Packages.callPackage (
+      {
+        buildPythonPackage,
+        pythonOlder,
+        parver,
+        pip,
+        pulumi,
+        semver,
+        setuptools,
+      }:
+      buildPythonPackage {
+        inherit
+          pname
+          meta
+          src
+          version
+          ;
+        format = "pyproject";
+
+        disabled = pythonOlder "3.7";
+
+        sourceRoot = "${src.name}/sdk/python";
+
+        propagatedBuildInputs = [
+          parver
+          pulumi
+          semver
+          setuptools
+        ];
+
+        postPatch = ''
+          if [[ -e "pyproject.toml" ]]; then
+            sed -i \
+              -e 's/^  version = .*/  version = "${version}"/g' \
+              pyproject.toml
+          else
+            sed -i \
+               -e 's/^VERSION = .*/VERSION = "${version}"/g' \
+               -e 's/^PLUGIN_VERSION = .*/PLUGIN_VERSION = "${version}"/g' \
+               setup.py
+          fi
+        '';
+
+        # Auto-generated; upstream does not have any tests.
+        # Verify that the version substitution works
+        checkPhase = ''
+          runHook preCheck
+
+          ${pip}/bin/pip show "${pname}" | grep "Version: ${version}" > /dev/null \
+            || (echo "ERROR: Version substitution seems to be broken"; exit 1)
+
+          runHook postCheck
+        '';
+
+        pythonImportsCheck = [
+          (builtins.replaceStrings [ "-" ] [ "_" ] pname)
+        ];
+      }
+    ) { };
+in
+{
+  owner,
+  repo,
+  rev,
+  version,
+  hash,
+  vendorHash,
+  cmdGen,
+  cmdRes,
+  extraLdflags,
+  env ? { },
+  meta,
+  fetchSubmodules ? false,
+  ...
+}@args:
+let
+  src = fetchFromGitHub {
+    name = "source-${repo}-${rev}";
+    inherit
+      owner
+      repo
+      rev
+      hash
+      fetchSubmodules
+      ;
+  };
+
+  pulumi-gen = mkBasePackage rec {
+    inherit
+      src
+      version
+      vendorHash
+      extraLdflags
+      env
+      ;
+
+    cmd = cmdGen;
+    pname = cmdGen;
+  };
+in
+mkBasePackage (
+  {
+    inherit
+      meta
+      src
+      version
+      vendorHash
+      extraLdflags
+      env
+      ;
+
+    pname = repo;
+
+    nativeBuildInputs = [
+      pulumi-gen
+    ];
+
+    cmd = cmdRes;
+
+    postConfigure = ''
+      pushd ..
+
+      chmod +w sdk/
+      ${cmdGen} schema
+
+      popd
+
+      VERSION=v${version} go generate cmd/${cmdRes}/main.go
+    '';
+
+    passthru.sdks.python = mkPythonPackage {
+      inherit meta src version;
+
+      pname = repo;
+    };
+  }
+  // args
+)

pkgs/by-name/pu/pulumi/extra/pulumi-test-hook.sh

@@ -0,0 +1,15 @@
+# shellcheck shell=bash
+
+appendToVar preCheckHooks pulumiTestHook
+
+pulumiTestHook() {
+    local tmpdir
+    tmpdir=$(mktemp -d)
+    export USER=${USER-nixbld} HOME=${HOME-$tmpdir}
+    export PULUMI_HOME=$tmpdir/.pulumi
+    export PULUMI_CONFIG_PASSPHRASE=5up3r53cr37
+    export PULUMI_SKIP_UPDATE_CHECK=1
+    export PULUMI_DISABLE_AUTOMATIC_PLUGIN_ACQUISITION=1
+    pulumi login "file://$tmpdir"
+    pulumi stack init -- "${pulumiStackName-nixpkgs}"
+}

pkgs/by-name/pu/pulumi/extra/test-resource-schema.nix

@@ -0,0 +1,35 @@
+{
+  lib,
+  runCommand,
+  pulumi,
+  jq,
+}:
+{
+  package,
+  name ? lib.removePrefix "pulumi-" (lib.getName package),
+  version ? lib.getVersion package,
+}:
+runCommand "pulumi-resource-${name}-schema-version-check"
+  {
+    resourceName = name;
+    expectedVersion = if version != null then version else "null";
+    nativeBuildInputs = [
+      jq
+      pulumi
+      package
+    ];
+    env = {
+      PULUMI_SKIP_UPDATE_CHECK = "1";
+      PULUMI_DISABLE_AUTOMATIC_PLUGIN_ACQUISITION = "1";
+    };
+    __darwinAllowLocalNetworking = true;
+    meta.timeout = 120;
+  }
+  ''
+    actualVersion=$(pulumi package get-schema -- "$resourceName" | jq -j .version)
+    if [[ $expectedVersion != "$actualVersion" ]]; then
+      echo "Expected schema version $expectedVersion, but got $actualVersion" >&2
+      false
+    fi
+    mkdir -p "$out"
+  ''

pkgs/by-name/pu/pulumi/package.nix

@@ -7,11 +7,9 @@
   git,
   buildPackages,
   # passthru
-  runCommand,
-  makeWrapper,
+  callPackage,
   testers,
   pulumi,
-  pulumiPackages,
 }:
 buildGoModule rec {
   pname = "pulumi";
@@ -117,25 +115,18 @@ buildGoModule rec {
   '';
 
   passthru = {
-    pkgs = pulumiPackages;
-    withPackages =
-      f:
-      runCommand "${pulumi.name}-with-packages"
-        {
-          nativeBuildInputs = [ makeWrapper ];
-        }
-        ''
-          mkdir -p $out/bin
-          makeWrapper ${pulumi}/bin/pulumi $out/bin/pulumi \
-            --suffix PATH : ${lib.makeBinPath (f pulumiPackages)} \
-            --set LD_LIBRARY_PATH "${lib.getLib stdenv.cc.cc}/lib"
-        '';
+    pkgs = callPackage ./plugins.nix { };
+    withPackages = callPackage ./with-packages.nix { };
     tests = {
       version = testers.testVersion {
         package = pulumi;
         version = "v${version}";
         command = "PULUMI_SKIP_UPDATE_CHECK=1 pulumi version";
       };
+      pulumiTestHookShellcheck = testers.shellcheck {
+        name = "pulumi-test-hook-shellcheck";
+        src = ./extra/pulumi-test-hook.sh;
+      };
     };
   };
 

pkgs/by-name/pu/pulumi/plugins.nix

@@ -0,0 +1,26 @@
+{
+  lib,
+  config,
+  generateSplicesForMkScope,
+  makeScopeWithSplicing',
+  attributePathToSplice ? [ "pulumiPackages" ],
+}:
+makeScopeWithSplicing' {
+  otherSplices = generateSplicesForMkScope attributePathToSplice;
+  extra = self: {
+    mkPulumiPackage = self.callPackage ./extra/mk-pulumi-package.nix { };
+    testResourceSchema = self.callPackage ./extra/test-resource-schema.nix { };
+    pulumiTestHook = ./extra/pulumi-test-hook.sh;
+  };
+  f =
+    self:
+    lib.packagesFromDirectoryRecursive {
+      inherit (self) callPackage;
+      directory = ./plugins;
+    }
+    // lib.optionalAttrs config.allowAliases {
+      pulumi-language-go = lib.warnOnInstantiate "pulumi-language-go has been renamed to pulumi-go" self.pulumi-go;
+      pulumi-language-nodejs = lib.warnOnInstantiate "pulumi-language-nodejs has been renamed to pulumi-nodejs" self.pulumi-nodejs;
+      pulumi-language-python = lib.warnOnInstantiate "pulumi-language-python has been renamed to pulumi-python" self.pulumi-python;
+    };
+}

pkgs/by-name/pu/pulumi/plugins/pulumi-go/package.nix

@@ -4,7 +4,7 @@
   pulumi,
 }:
 buildGoModule rec {
-  pname = "pulumi-language-go";
+  pname = "pulumi-go";
   inherit (pulumi) version src;
 
   sourceRoot = "${src.name}/sdk/go/pulumi-language-go";

pkgs/by-name/pu/pulumi/plugins/pulumi-nodejs/package.nix

@@ -7,7 +7,7 @@
   python3,
 }:
 buildGoModule rec {
-  pname = "pulumi-language-nodejs";
+  pname = "pulumi-nodejs";
   inherit (pulumi) version src;
 
   sourceRoot = "${src.name}/sdk/nodejs/cmd/pulumi-language-nodejs";

pkgs/by-name/pu/pulumi/plugins/pulumi-python/package.nix

@@ -1,12 +1,13 @@
 {
   lib,
   buildGoModule,
+  callPackage,
   pulumi,
   bash,
   python3,
 }:
 buildGoModule rec {
-  pname = "pulumi-language-python";
+  pname = "pulumi-python";
   inherit (pulumi) version src;
 
   sourceRoot = "${src.name}/sdk/python/cmd/pulumi-language-python";
@@ -45,6 +46,8 @@ buildGoModule rec {
       ../../dist/pulumi-analyzer-policy-python
   '';
 
+  passthru.tests.smokeTest = callPackage ./smoke-test/default.nix { };
+
   meta = {
     homepage = "https://www.pulumi.com/docs/iac/languages-sdks/python/";
     description = "Language host for Pulumi programs written in Python";

pkgs/by-name/pu/pulumi/plugins/pulumi-python/smoke-test/Pulumi.yaml

@@ -0,0 +1,2 @@
+name: smoke-test
+runtime: python

pkgs/by-name/pu/pulumi/plugins/pulumi-python/smoke-test/__main__.py

@@ -0,0 +1,18 @@
+from binascii import b2a_hex
+from os import urandom
+
+from pulumi import export, ResourceOptions
+from pulumi.dynamic import Resource, ResourceProvider, CreateResult
+
+
+class RandomProvider(ResourceProvider):
+    def create(self, inputs):
+        return CreateResult(b2a_hex(urandom(16)), outs={})
+
+
+class Random(Resource):
+    def __init__(self, name, opts = None):
+         super().__init__(RandomProvider(), name, {}, opts)
+
+
+export("out", Random(name="random_test").id)

pkgs/by-name/pu/pulumi/plugins/pulumi-python/smoke-test/default.nix

@@ -0,0 +1,37 @@
+{
+  lib,
+  stdenvNoCC,
+  pulumiTestHook,
+  pulumi,
+  pulumi-python,
+  python3Packages,
+}:
+stdenvNoCC.mkDerivation {
+  name = "pulumi-python-smoke-test";
+  src = builtins.filterSource (name: _: !(lib.hasSuffix ".nix" name)) ./.;
+
+  doCheck = true;
+
+  nativeCheckInputs = [
+    pulumiTestHook
+    pulumi
+    pulumi-python
+    python3Packages.pulumi
+  ];
+
+  __darwinAllowLocalNetworking = true;
+
+  checkPhase = ''
+    runHook preCheck
+    pulumi update --skip-preview
+    stackOutput=$(pulumi stack output out)
+    [[ $stackOutput =~ ^[0-9a-f]{32}$ ]]
+    runHook postCheck
+  '';
+
+  installPhase = ''
+    runHook preInstall
+    mkdir -p "$out"
+    runHook postInstall
+  '';
+}

pkgs/by-name/pu/pulumi/with-packages.nix

@@ -0,0 +1,35 @@
+{
+  lib,
+  runCommand,
+  pulumi,
+  pulumiPackages,
+  makeBinaryWrapper,
+}:
+f:
+# Note that Pulumi prints a warning for “ambient” plugins (i.e. from PATH). E.g.
+#   warning: using pulumi-resource-* from $PATH at /nix/store/*
+# See also https://github.com/pulumi/pulumi/issues/14525
+#
+# We can install packages alongside pulumi executable, but that would only
+# suppress the warning for packages that are bundled with the official
+# distribution. Refer to the implementation of workspace.IsPluginBundled:
+# https://pkg.go.dev/github.com/pulumi/pulumi/sdk/v3@v3.150.0/go/common/workspace#IsPluginBundled
+# https://github.com/pulumi/pulumi/blob/v3.150.0/sdk/go/common/workspace/plugins.go#L2202-L2210
+runCommand "pulumi-with-packages"
+  {
+    inherit pulumi;
+    makeWrapperArgs = [
+      "--suffix"
+      "PATH"
+      ":"
+      (lib.makeBinPath (f pulumiPackages))
+    ];
+    __structuredAttrs = true;
+    nativeBuildInputs = [ makeBinaryWrapper ];
+    meta.mainProgram = "pulumi";
+  }
+  ''
+    mkdir -p "$out/bin"
+    ln -s -t "$out" "$pulumi/share"
+    makeWrapper "$pulumi/bin/pulumi" "$out/bin/pulumi" "''${makeWrapperArgs[@]}"
+  ''

pkgs/development/python-modules/pulumi/default.nix

@@ -20,7 +20,7 @@
 }:
 let
   inherit (pkgs.pulumi) pname version src;
-  inherit (pulumiPackages) pulumi-language-python;
+  inherit (pulumiPackages) pulumi-python;
   sourceRoot = "${src.name}/sdk/python";
 in
 buildPythonPackage {
@@ -63,7 +63,7 @@ buildPythonPackage {
     pytest
     pytest-asyncio
     pytest-timeout
-    pulumi-language-python
+    pulumi-python
   ];
 
   # https://github.com/pulumi/pulumi/blob/0acaf8060640fdd892abccf1ce7435cd6aae69fe/sdk/python/scripts/test_fast.sh#L10-L11

pkgs/tools/admin/pulumi-packages/base.nix

@@ -1,137 +0,0 @@
-{ buildGoModule
-, fetchFromGitHub
-, python3Packages
-}:
-let
-  mkBasePackage =
-    { pname
-    , src
-    , version
-    , vendorHash
-    , cmd
-    , extraLdflags
-    , env
-    , ...
-    }@args: buildGoModule (rec {
-      inherit pname src vendorHash version env;
-
-      sourceRoot = "${src.name}/provider";
-
-      subPackages = [ "cmd/${cmd}" ];
-
-      doCheck = false;
-
-      ldflags = [
-        "-s"
-        "-w"
-      ] ++ extraLdflags;
-    } // args);
-
-  mkPythonPackage =
-    { meta
-    , pname
-    , src
-    , version
-    , ...
-    }: python3Packages.callPackage
-      ({ buildPythonPackage, pythonOlder, parver, pip, pulumi, semver, setuptools }:
-      buildPythonPackage {
-        inherit pname meta src version;
-        format = "pyproject";
-
-        disabled = pythonOlder "3.7";
-
-        sourceRoot = "${src.name}/sdk/python";
-
-        propagatedBuildInputs = [
-          parver
-          pulumi
-          semver
-          setuptools
-        ];
-
-        postPatch = ''
-          if [[ -e "pyproject.toml" ]]; then
-            sed -i \
-              -e 's/^  version = .*/  version = "${version}"/g' \
-              pyproject.toml
-          else
-            sed -i \
-               -e 's/^VERSION = .*/VERSION = "${version}"/g' \
-               -e 's/^PLUGIN_VERSION = .*/PLUGIN_VERSION = "${version}"/g' \
-               setup.py
-          fi
-        '';
-
-        # Auto-generated; upstream does not have any tests.
-        # Verify that the version substitution works
-        checkPhase = ''
-          runHook preCheck
-
-          ${pip}/bin/pip show "${pname}" | grep "Version: ${version}" > /dev/null \
-            || (echo "ERROR: Version substitution seems to be broken"; exit 1)
-
-          runHook postCheck
-        '';
-
-        pythonImportsCheck = [
-          (builtins.replaceStrings [ "-" ] [ "_" ] pname)
-        ];
-      })
-      { };
-in
-{ owner
-, repo
-, rev
-, version
-, hash
-, vendorHash
-, cmdGen
-, cmdRes
-, extraLdflags
-, env ? { }
-, meta
-, fetchSubmodules ? false
-, ...
-}@args:
-let
-  src = fetchFromGitHub {
-    name = "source-${repo}-${rev}";
-    inherit owner repo rev hash fetchSubmodules;
-  };
-
-  pulumi-gen = mkBasePackage rec {
-    inherit src version vendorHash extraLdflags env;
-
-    cmd = cmdGen;
-    pname = cmdGen;
-  };
-in
-mkBasePackage ({
-  inherit meta src version vendorHash extraLdflags env;
-
-  pname = repo;
-
-  nativeBuildInputs = [
-    pulumi-gen
-  ];
-
-  cmd = cmdRes;
-
-  postConfigure = ''
-    pushd ..
-
-    chmod +w sdk/
-    ${cmdGen} schema
-
-    popd
-
-    VERSION=v${version} go generate cmd/${cmdRes}/main.go
-  '';
-
-  passthru.sdks.python = mkPythonPackage {
-    inherit meta src version;
-
-    pname = repo;
-  };
-} // args)

pkgs/tools/admin/pulumi-packages/default.nix

@@ -1,16 +0,0 @@
-{ callPackage }:
-let
-  mkPulumiPackage = callPackage ./base.nix { };
-  callPackage' = p: args: callPackage p (args // { inherit mkPulumiPackage; });
-in
-{
-  pulumi-aws-native = callPackage' ./pulumi-aws-native.nix { };
-  pulumi-azure-native = callPackage' ./pulumi-azure-native.nix { };
-  pulumi-command = callPackage' ./pulumi-command.nix { };
-  pulumi-hcloud = callPackage' ./pulumi-hcloud.nix { };
-  pulumi-language-go = callPackage ./pulumi-language-go.nix { };
-  pulumi-language-nodejs = callPackage ./pulumi-language-nodejs.nix { };
-  pulumi-language-python = callPackage ./pulumi-language-python.nix { };
-  pulumi-random = callPackage' ./pulumi-random.nix { };
-  pulumi-yandex-unofficial = callPackage' ./pulumi-yandex-unofficial.nix { };
-}

pkgs/top-level/all-packages.nix

@@ -4405,9 +4405,7 @@ with pkgs;
 
   pipecontrol = libsForQt5.callPackage ../applications/audio/pipecontrol { };
 
-  pulumiPackages = recurseIntoAttrs (
-    callPackage ../tools/admin/pulumi-packages { }
-  );
+  pulumiPackages = recurseIntoAttrs pulumi.pkgs;
 
   pulumi-bin = callPackage ../tools/admin/pulumi-bin { };