diff --git a/ci/OWNERS b/ci/OWNERS index 43ef6f666651..463c546e695b 100644 --- a/ci/OWNERS +++ b/ci/OWNERS @@ -39,6 +39,7 @@ /lib/derivations.nix @NixOS/stdenv /lib/fetchers.nix @alyssais @NixOS/stdenv /lib/meta.nix @alyssais @NixOS/stdenv +/lib/meta-types.nix @infinisil @adisbladis @NixOS/stdenv /lib/source-types.nix @alyssais @NixOS/stdenv /lib/systems @alyssais @NixOS/stdenv ## Libraries / Module system @@ -67,7 +68,6 @@ /pkgs/stdenv/generic/problems.nix @infinisil /pkgs/test/problems @infinisil /pkgs/stdenv/generic/check-meta.nix @infinisil @Ericson2314 @adisbladis @NixOS/stdenv -/pkgs/stdenv/generic/meta-types.nix @infinisil @adisbladis @NixOS/stdenv /pkgs/stdenv/cross @Ericson2314 @NixOS/stdenv /pkgs/build-support @philiptaron /pkgs/build-support/cc-wrapper @Ericson2314 diff --git a/lib/attrsets.nix b/lib/attrsets.nix index 7dae595c9e39..811225c542f1 100644 --- a/lib/attrsets.nix +++ b/lib/attrsets.nix @@ -1800,22 +1800,28 @@ rec { ::: */ matchAttrs = - pattern: attrs: + let + recurse = + pattern: attrs: + all ( + # Compare equality between `pattern` & `attrs`. + attr: + # Missing attr, not equal. + attrs ? ${attr} + && ( + let + lhs = pattern.${attr}; + rhs = attrs.${attr}; + in + # Simple equality check is primarily for non-attrsets, but we run it + # on attrsets too, since it may let us avoid recursing + lhs == rhs || isAttrs lhs && isAttrs rhs && recurse lhs rhs + ) + ) (attrNames pattern); + in + pattern: assert isAttrs pattern; - all ( - # Compare equality between `pattern` & `attrs`. - attr: - # Missing attr, not equal. - attrs ? ${attr} - && ( - let - lhs = pattern.${attr}; - rhs = attrs.${attr}; - in - # If attrset check recursively - if isAttrs lhs then isAttrs rhs && matchAttrs lhs rhs else lhs == rhs - ) - ) (attrNames pattern); + recurse pattern; /** Override only the attributes that are already present in the old set diff --git a/pkgs/stdenv/generic/meta-types.nix b/lib/meta-types.nix similarity index 94% rename from pkgs/stdenv/generic/meta-types.nix rename to lib/meta-types.nix index 1396243e3002..1d67f5f063e8 100644 --- a/pkgs/stdenv/generic/meta-types.nix +++ b/lib/meta-types.nix @@ -1,6 +1,7 @@ { lib }: # Simple internal type checks for meta. -# This file is not a stable interface and may be changed arbitrarily. +# This file is only intended for internal nixpkgs use. +# It is not a stable interface and may be changed arbitrarily. # # TODO: add a method to the module system types # see https://github.com/NixOS/nixpkgs/pull/273935#issuecomment-1854173100 @@ -172,10 +173,10 @@ lib.fix (self: { enum = values: - assert isList values && all isString values; + assert isList values; { - name = "enum<${concatStringsSep "," values}>"; - verify = v: isString v && elem v values; + name = if all isString values then "enum<${concatStringsSep "," values}>" else "enum"; + verify = v: elem v values; }; record = diff --git a/lib/systems/doubles.nix b/lib/systems/doubles.nix index e9fa908dfc88..d3e184320fbd 100644 --- a/lib/systems/doubles.nix +++ b/lib/systems/doubles.nix @@ -1,7 +1,15 @@ { lib }: let - inherit (lib) lists; + inherit (lib) + lists + splitString + ; inherit (lib.systems) parse; + inherit (parse) + mkSystemFromSkeleton + mkSkeletonFromList + doubleFromSystem + ; inherit (lib.systems.inspect) predicates; inherit (lib.attrsets) matchAttrs; @@ -117,9 +125,17 @@ let "x86_64-uefi" ]; - allParsed = map parse.mkSystemFromString all; + uncheckedSystemFromString = + let + systemType = { + _type = "system"; + }; + in + s: mkSystemFromSkeleton (mkSkeletonFromList (splitString "-" s)) // systemType; - filterDoubles = f: map parse.doubleFromSystem (lists.filter f allParsed); + allParsed = map uncheckedSystemFromString all; + + filterDoubles = f: map doubleFromSystem (lists.filter f allParsed); in { diff --git a/lib/systems/inspect.nix b/lib/systems/inspect.nix index 36a4036f210a..98b3fc050bc1 100644 --- a/lib/systems/inspect.nix +++ b/lib/systems/inspect.nix @@ -2,14 +2,17 @@ let inherit (lib) + all any + attrNames attrValues concatMap filter + flip hasPrefix + isAttrs isList mapAttrs - matchAttrs recursiveUpdateUntil toList ; @@ -24,19 +27,49 @@ let execFormats ; - abis = mapAttrs (_: abi: removeAttrs abi [ "assertions" ]) lib.systems.parse.abis; + # Based on lib.attrsets.matchAttrs, but with: + # - the initial isAttrs assertion removed, since this function is only ever + # called with attrsets + # - isAttrs only performed on one side when recursing, since our input data + # will always share a structure + matchAttrsUnchecked = + pattern: attrs: + all ( + # Compare equality between `pattern` & `attrs`. + attr: + # Missing attr, not equal. + attrs ? ${attr} + && ( + let + lhs = pattern.${attr}; + rhs = attrs.${attr}; + in + # Simple equality check is primarily for non-attrsets, but we run it + # on attrsets too, since it may let us avoid recursing + lhs == rhs || isAttrs lhs && matchAttrsUnchecked lhs rhs + ) + ) (attrNames pattern); + + removeAssertions = flip removeAttrs [ "assertions" ]; + abis = mapAttrs ( + _: abi: if abi ? assertions then removeAssertions abi else abi + ) lib.systems.parse.abis; in rec { # these patterns are to be matched against {host,build,target}Platform.parsed + # + # Note: All toplevel attributes within a pattern are expected to be attrsets. + # matchAttrsUnchecked should be changed if a pattern is ever added that + # doesn't follow this axiom patterns = rec { # The patterns below are lists in sum-of-products form. # # Each attribute is list of product conditions; non-list values are treated # as a singleton list. If *any* product condition in the list matches then # the predicate matches. Each product condition is tested by - # `lib.attrsets.matchAttrs`, which requires a match on *all* attributes of - # the product. + # `matchAttrsUnchecked`, which requires a match on *all* attributes of the + # product. isi686 = { cpu = cpuTypes.i686; @@ -489,14 +522,40 @@ rec { ) pat2 ) pat1; - matchAnyAttrs = - patterns: - if isList patterns then - attrs: any (pattern: matchAttrs pattern attrs) patterns - else - matchAttrs patterns; + matchAnyPattern = + let + # same as matchAttrsUnchecked definition at the top of the file, but: + # - pattern names are cached and reused for multiple attrset calls + # - avoid running isAttrs since all patterns are nested attrsets + matchPattern = + pattern: + let + names = attrNames pattern; + in + attrs: + all ( + attr: + attrs ? ${attr} + && ( + let + lhs = pattern.${attr}; + rhs = attrs.${attr}; + in + lhs == rhs || matchAttrsUnchecked lhs rhs + ) + ) names; - predicates = mapAttrs (_: matchAnyAttrs) patterns; + in + pattern: + if isList pattern then + let + cachedPatterns = map matchPattern pattern; + in + attrs: any (pattern: pattern attrs) cachedPatterns + else + matchPattern pattern; + + predicates = mapAttrs (_: matchAnyPattern) patterns; # these patterns are to be matched against the entire # {host,build,target}Platform structure; they include a `parsed={}` marker so diff --git a/lib/systems/parse.nix b/lib/systems/parse.nix index 98db2595235f..cdfeafe7c18d 100644 --- a/lib/systems/parse.nix +++ b/lib/systems/parse.nix @@ -17,6 +17,9 @@ { lib }: let + inherit (import ../meta-types.nix { inherit lib; }) + enum + ; inherit (lib) all any @@ -28,7 +31,6 @@ let id length mapAttrs - mergeOneOption optionalString splitString versionAtLeast @@ -46,20 +48,19 @@ let isCygwin ; - inherit (lib.types) - enum - isType - mkOptionType - setType - ; - setTypes = type: - mapAttrs ( - name: value: - assert type.check value; - setType type.name ({ inherit name; } // value) - ); + if type ? verify then + let + inherit (type) verify; + in + mapAttrs ( + name: value: + assert verify value; + { inherit name; } // value + ) + else + mapAttrs (name: value: { inherit name; } // value); # gnu-config will ignore the portion of a triple matching the # regex `e?abi.*$` when determining the validity of a triple. In @@ -76,11 +77,9 @@ in rec { ################################################################################ - - types.openSignificantByte = mkOptionType { + types.openSignificantByte = { name = "significant-byte"; description = "Endianness"; - merge = mergeOneOption; }; types.significantByte = enum (attrValues significantBytes); @@ -103,21 +102,24 @@ rec { ################################################################################ - types.openCpuType = mkOptionType { + types.openCpuType = { name = "cpu-type"; description = "instruction set architecture name and information"; - merge = mergeOneOption; - check = - x: - types.bitWidth.check x.bits - && (if 8 < x.bits then types.significantByte.check x.significantByte else !(x ? significantByte)); + verify = + let + verifyBitWidth = types.bitWidth.verify; + verifySignificantByte = types.significantByte.verify; + in + v: + verifyBitWidth v.bits + && (if 8 < v.bits then verifySignificantByte v.significantByte else !(v ? significantByte)); }; types.cpuType = enum (attrValues cpuTypes); cpuTypes = let - inherit (significantBytes) bigEndian littleEndian; + inherit (significantBytes) littleEndian bigEndian; in setTypes types.openCpuType { arm = { @@ -487,10 +489,9 @@ rec { ################################################################################ - types.openVendor = mkOptionType { + types.openVendor = { name = "vendor"; description = "vendor for the platform"; - merge = mergeOneOption; }; types.vendor = enum (attrValues vendors); @@ -499,23 +500,19 @@ rec { apple = { }; pc = { }; knuth = { }; - # Actually matters, unlocking some MinGW-w64-specific options in GCC. See # bottom of https://sourceforge.net/p/mingw-w64/wiki2/Unicode%20apps/ w64 = { }; - none = { }; unknown = { }; }; ################################################################################ - types.openExecFormat = mkOptionType { + types.openExecFormat = { name = "exec-format"; description = "executable container used by the kernel"; - merge = mergeOneOption; }; - types.execFormat = enum (attrValues execFormats); execFormats = setTypes types.openExecFormat { @@ -524,16 +521,14 @@ rec { macho = { }; pe = { }; wasm = { }; - unknown = { }; }; ################################################################################ - types.openKernelFamily = mkOptionType { + types.openKernelFamily = { name = "exec-format"; description = "executable container used by the kernel"; - merge = mergeOneOption; }; types.kernelFamily = enum (attrValues kernelFamilies); @@ -545,12 +540,15 @@ rec { ################################################################################ - types.openKernel = mkOptionType { - name = "kernel"; + types.openKernel = { + name = "open-kernel"; description = "kernel name and information"; - merge = mergeOneOption; - check = - x: types.execFormat.check x.execFormat && all types.kernelFamily.check (attrValues x.families); + verify = + let + verifyExecFormat = types.execFormat.verify; + verifyKernelFamily = types.kernelFamily.verify; + in + v: verifyExecFormat v.execFormat && all verifyKernelFamily (attrValues v.families); }; types.kernel = enum (attrValues kernels); @@ -647,10 +645,9 @@ rec { ################################################################################ - types.openAbi = mkOptionType { + types.openAbi = { name = "abi"; description = "binary interface for compiled code and syscalls"; - merge = mergeOneOption; }; types.abi = enum (attrValues abis); @@ -760,119 +757,138 @@ rec { ################################################################################ - types.parsedPlatform = mkOptionType { + types.parsedPlatform = { name = "system"; description = "fully parsed representation of llvm- or nix-style platform tuple"; - merge = mergeOneOption; - check = + verify = + let + verifyCpu = types.cpuType.verify; + verifyVendor = types.vendor.verify; + verifyKernel = types.kernel.verify; + verifyAbi = types.abi.verify; + in { cpu, vendor, kernel, abi, }: - types.cpuType.check cpu - && types.vendor.check vendor - && types.kernel.check kernel - && types.abi.check abi; + verifyCpu cpu && verifyVendor vendor && verifyKernel kernel && verifyAbi abi; }; - isSystem = isType "system"; + isSystem = v: v._type or null == "system"; mkSystem = + let + inherit (types.parsedPlatform) verify; + in components: - assert types.parsedPlatform.check components; - setType "system" components; + assert verify components; + components + // { + _type = "system"; + }; mkSkeletonFromList = + let + linuxComponents = [ + "eabi" + "eabihf" + "elf" + "gnu" + ]; + appleComponents = [ + "redox" + "mmixware" + "ghcjs" + "mingw32" + "uefi" + ]; + in l: { "1" = - if head l == "avr" then + let + firstComponent = head l; + in + if firstComponent == "avr" then { - cpu = head l; + cpu = firstComponent; kernel = "none"; abi = "unknown"; } else throw "system string '${lib.concatStringsSep "-" l}' with 1 component is ambiguous"; "2" = # We only do 2-part hacks for things Nix already supports - if elemAt l 1 == "cygwin" then + let + secondComponent = elemAt l 1; + in + if secondComponent == "cygwin" then mkSkeletonFromList [ (head l) "pc" - "cygwin" + secondComponent ] # MSVC ought to be the default ABI so this case isn't needed. But then it # becomes difficult to handle the gnu* variants for Aarch32 correctly for # minGW. So it's easier to make gnu* the default for the MinGW, but # hack-in MSVC for the non-MinGW case right here. - else if elemAt l 1 == "windows" then + else if secondComponent == "windows" then { cpu = head l; - kernel = "windows"; + kernel = secondComponent; abi = "msvc"; } - else if (elemAt l 1) == "elf" then + else if secondComponent == "elf" then { cpu = head l; vendor = "unknown"; kernel = "none"; - abi = elemAt l 1; + abi = secondComponent; } else { cpu = head l; - kernel = elemAt l 1; + kernel = secondComponent; }; "3" = + let + secondComponent = elemAt l 1; + thirdComponent = elemAt l 2; + in # cpu-kernel-environment - if - elemAt l 1 == "linux" - || elem (elemAt l 2) [ - "eabi" - "eabihf" - "elf" - "gnu" - ] - then + if secondComponent == "linux" || elem thirdComponent linuxComponents then { cpu = head l; - kernel = elemAt l 1; - abi = elemAt l 2; + kernel = secondComponent; + abi = thirdComponent; vendor = "unknown"; } # cpu-vendor-os else if - elemAt l 1 == "apple" - || elem (elemAt l 2) [ - "redox" - "mmixware" - "ghcjs" - "mingw32" - "uefi" - ] - || hasPrefix "freebsd" (elemAt l 2) - || hasPrefix "netbsd" (elemAt l 2) - || hasPrefix "openbsd" (elemAt l 2) - || hasPrefix "genode" (elemAt l 2) - || hasPrefix "wasm32" (elemAt l 0) + secondComponent == "apple" + || elem thirdComponent appleComponents + || hasPrefix "freebsd" thirdComponent + || hasPrefix "netbsd" thirdComponent + || hasPrefix "openbsd" thirdComponent + || hasPrefix "genode" thirdComponent + || hasPrefix "wasm32" (head l) then { cpu = head l; - vendor = elemAt l 1; + vendor = secondComponent; kernel = - if elemAt l 2 == "mingw32" then + if thirdComponent == "mingw32" then "windows" # autotools breaks on -gnu for window else - elemAt l 2; + thirdComponent; } # lots of tools expect a triplet for Cygwin, even though the vendor is just "pc" - else if elemAt l 2 == "cygwin" then + else if thirdComponent == "cygwin" then { cpu = head l; - vendor = elemAt l 1; - kernel = "cygwin"; + vendor = secondComponent; + kernel = thirdComponent; } else throw "system string '${lib.concatStringsSep "-" l}' with 3 components is ambiguous"; @@ -887,7 +903,17 @@ rec { or (throw "system string '${lib.concatStringsSep "-" l}' has invalid number of hyphen-separated components"); # This should revert the job done by config.guess from the gcc compiler. + # Note: this does _not_ verify that the system is valid + # `mkSystemFromString` is recommended for external use mkSystemFromSkeleton = + let + getCpu = name: cpuTypes.${name} or (throw "Unknown CPU type: ${name}"); + getVendor = name: vendors.${name} or (throw "Unknown vendor: ${name}"); + getKernel = name: kernels.${name} or (throw "Unknown kernel: ${name}"); + getAbi = name: abis.${name} or (throw "Unknown ABI: ${name}"); + hasDarwinPrefix = hasPrefix "darwin"; + hasBsdPrefix = hasPrefix "netbsd"; + in { cpu, # Optional, but fallback too complex for here. @@ -902,11 +928,6 @@ rec { null, }@args: let - getCpu = name: cpuTypes.${name} or (throw "Unknown CPU type: ${name}"); - getVendor = name: vendors.${name} or (throw "Unknown vendor: ${name}"); - getKernel = name: kernels.${name} or (throw "Unknown kernel: ${name}"); - getAbi = name: abis.${name} or (throw "Unknown ABI: ${name}"); - parsed = { cpu = getCpu args.cpu; vendor = @@ -919,10 +940,10 @@ rec { else vendors.unknown; kernel = - if hasPrefix "darwin" args.kernel then - getKernel "darwin" - else if hasPrefix "netbsd" args.kernel then - getKernel "netbsd" + if hasDarwinPrefix args.kernel then + kernels.darwin + else if hasBsdPrefix args.kernel then + kernels.netbsd else getKernel (removeAbiSuffix args.kernel); abi = @@ -941,11 +962,12 @@ rec { }; in - mkSystem parsed; + parsed; - mkSystemFromString = s: mkSystemFromSkeleton (mkSkeletonFromList (splitString "-" s)); + mkSystemFromString = s: mkSystem (mkSystemFromSkeleton (mkSkeletonFromList (splitString "-" s))); - kernelName = kernel: kernel.name + toString (kernel.version or ""); + kernelName = + kernel: if kernel ? version then kernel.name + toString kernel.version else kernel.name; darwinArch = cpu: if cpu.name == "aarch64" then "arm64" else cpu.name; diff --git a/lib/tests/systems.nix b/lib/tests/systems.nix index 3feef87d97c0..e2b9093511c6 100644 --- a/lib/tests/systems.nix +++ b/lib/tests/systems.nix @@ -215,6 +215,22 @@ lib.runTests ( }) // { + test_platforms_pass_typecheck = { + # To improve performance, the result of parsing all 70+ systems in + # `lib.platforms` into their attrset representations aren't typechecked. + # The results are expected to be constant, and avoiding the slow + # validation gives a meaningful improvement to evaluation speed. We ensure + # that all systems pass validation here + expr = builtins.filter ( + system: + let + evalResult = builtins.tryEval (lib.systems.parse.mkSystemFromString system); + in + evalResult.success == false + ) lib.platforms.all; + + expected = [ ]; + }; test_equals_example_x86_64-linux = { expr = lib.systems.equals (lib.systems.elaborate "x86_64-linux") ( lib.systems.elaborate "x86_64-linux" diff --git a/pkgs/stdenv/generic/check-meta.nix b/pkgs/stdenv/generic/check-meta.nix index 70c009c3e83a..780e98ed3adc 100644 --- a/pkgs/stdenv/generic/check-meta.nix +++ b/pkgs/stdenv/generic/check-meta.nix @@ -300,7 +300,7 @@ let metaType = let - types = import ./meta-types.nix { inherit lib; }; + types = import ../../../lib/meta-types.nix { inherit lib; }; inherit (types) str union diff --git a/pkgs/stdenv/generic/problems.nix b/pkgs/stdenv/generic/problems.nix index 0f03c393e384..114d586c0df6 100644 --- a/pkgs/stdenv/generic/problems.nix +++ b/pkgs/stdenv/generic/problems.nix @@ -242,7 +242,7 @@ rec { # The type for meta.problems problemsType = let - types = import ./meta-types.nix { inherit lib; }; + types = import ../../../lib/meta-types.nix { inherit lib; }; inherit (types) str listOf