diff --git a/lib/systems/platforms.nix b/lib/systems/platforms.nix
index b10c787728a0..7c49b40e53d4 100644
--- a/lib/systems/platforms.nix
+++ b/lib/systems/platforms.nix
@@ -152,11 +152,6 @@ rec {
       DTB = true;
       autoModules = true;
       preferBuiltin = true;
-      extraConfig = ''
-        # The default (=y) forces us to have the XHCI firmware available in initrd,
-        # which our initrd builder can't currently do easily.
-        USB_XHCI_TEGRA m
-      '';
       target = "Image";
     };
     gcc = {
diff --git a/pkgs/os-specific/linux/kernel/common-config.nix b/pkgs/os-specific/linux/kernel/common-config.nix
index 77639423ffe5..f46e25f2c6be 100644
--- a/pkgs/os-specific/linux/kernel/common-config.nix
+++ b/pkgs/os-specific/linux/kernel/common-config.nix
@@ -676,6 +676,10 @@ let
       USB_DWC3_DUAL_ROLE = yes;
 
       USB_XHCI_SIDEBAND = whenAtLeast "6.16" yes; # needed for audio offload
+
+      # The default (=y) forces us to have the XHCI firmware available in initrd,
+      # which our initrd builder can't currently do easily.
+      USB_XHCI_TEGRA = lib.mkIf stdenv.hostPlatform.isAarch64 module;
     };
 
     usb-serial = {