From febff1dccd2c173472fe4a6bed2e620429c5b1ba Mon Sep 17 00:00:00 2001 From: Jacob Abel Date: Sat, 21 May 2022 22:34:11 -0400 Subject: [PATCH 1/5] lib/strings: allow toInt to parse zero-padded strings --- lib/strings.nix | 20 ++++++++++++++++---- lib/tests/misc.nix | 28 ++++++++++++++++++++++++++++ 2 files changed, 44 insertions(+), 4 deletions(-) diff --git a/lib/strings.nix b/lib/strings.nix index be217cb06469..8f3568fc1fc5 100644 --- a/lib/strings.nix +++ b/lib/strings.nix @@ -792,15 +792,27 @@ rec { => 1337 toInt "-4" => -4 + toInt " 123 " + => 123 + toInt "00024" + => 24 toInt "3.14" => error: floating point JSON numbers are not supported */ # Obviously, it is a bit hacky to use fromJSON this way. toInt = str: - let may_be_int = fromJSON str; in - if isInt may_be_int - then may_be_int - else throw "Could not convert ${str} to int."; + let + strippedInput = match "[[:space:]]*(0*)(.*)" str; + isNonZeroEmpty = match "[[:space:]]*" (lib.last strippedInput) == []; + isZeroNonEmpty = head strippedInput != ""; + mayBeInt = fromJSON (lib.last strippedInput); + in + if isNonZeroEmpty && isZeroNonEmpty + then 0 + else + if isInt mayBeInt + then mayBeInt + else throw "Could not convert ${str} to int."; /* Read a list of paths from `file`, relative to the `rootPath`. Lines beginning with `#` are treated as comments and ignored. diff --git a/lib/tests/misc.nix b/lib/tests/misc.nix index 8e0cf1f45bb6..ef4483219f7e 100644 --- a/lib/tests/misc.nix +++ b/lib/tests/misc.nix @@ -327,6 +327,34 @@ runTests { expected = "Hello\\x20World"; }; + testToInt = testAllTrue [ + # Naive + (123 == toInt "123") + (0 == toInt "0") + # Whitespace Padding + (123 == toInt " 123") + (123 == toInt "123 ") + (123 == toInt " 123 ") + (123 == toInt " 123 ") + (0 == toInt " 0") + (0 == toInt "0 ") + (0 == toInt " 0 ") + # Zero Padding + (123 == toInt "0123") + (123 == toInt "0000123") + (0 == toInt "000000") + # Whitespace and Zero Padding + (123 == toInt " 0123") + (123 == toInt "0123 ") + (123 == toInt " 0123 ") + (123 == toInt " 0000123") + (123 == toInt "0000123 ") + (123 == toInt " 0000123 ") + (0 == toInt " 000000") + (0 == toInt "000000 ") + (0 == toInt " 000000 ") + ]; + # LISTS testFilter = { From 3d196a5f2a72595b14c439a9b4aba7737c0f1ebe Mon Sep 17 00:00:00 2001 From: Jacob Abel Date: Mon, 23 May 2022 03:39:48 -0400 Subject: [PATCH 2/5] lib/strings: Update toInt to handle intermixed ws and zeros. Added tests --- lib/strings.nix | 20 +++++++++++++------- lib/tests/misc.nix | 11 +++++++++++ lib/tests/modules.sh | 2 +- 3 files changed, 25 insertions(+), 8 deletions(-) diff --git a/lib/strings.nix b/lib/strings.nix index 8f3568fc1fc5..c6269e755e2a 100644 --- a/lib/strings.nix +++ b/lib/strings.nix @@ -802,16 +802,22 @@ rec { # Obviously, it is a bit hacky to use fromJSON this way. toInt = str: let - strippedInput = match "[[:space:]]*(0*)(.*)" str; - isNonZeroEmpty = match "[[:space:]]*" (lib.last strippedInput) == []; - isZeroNonEmpty = head strippedInput != ""; - mayBeInt = fromJSON (lib.last strippedInput); + # RegEx: Match any leading whitespace, then any zero padding, and capture any remaining + # digits after that, and finally match any trailing whitespace. + strippedInput = match "[[:space:]]*0*([[:digit:]]+)[[:space:]]*" str; + + # RegEx: Match any leading whitespace, at least one '0', and any trailing whitespace. + isZero = match "[[:space:]]*0+[[:space:]]*" str == []; + + # Attempt to parse input + parsedInput = fromJSON (elemAt strippedInput 0); in - if isNonZeroEmpty && isZeroNonEmpty + # Value is zero + if isZero then 0 else - if isInt mayBeInt - then mayBeInt + if strippedInput != null && isInt parsedInput + then parsedInput else throw "Could not convert ${str} to int."; /* Read a list of paths from `file`, relative to the `rootPath`. diff --git a/lib/tests/misc.nix b/lib/tests/misc.nix index ef4483219f7e..97d53026c644 100644 --- a/lib/tests/misc.nix +++ b/lib/tests/misc.nix @@ -355,6 +355,17 @@ runTests { (0 == toInt " 000000 ") ]; + testToIntFails = testAllTrue [ + ( builtins.tryEval (toInt "") == { success = false; value = false; } ) + ( builtins.tryEval (toInt "123 123") == { success = false; value = false; } ) + ( builtins.tryEval (toInt "0 123") == { success = false; value = false; } ) + ( builtins.tryEval (toInt " 0d ") == { success = false; value = false; } ) + ( builtins.tryEval (toInt " foo ") == { success = false; value = false; } ) + ( builtins.tryEval (toInt " foo 123 ") == { success = false; value = false; } ) + ( builtins.tryEval (toInt " foo 00123 ") == { success = false; value = false; } ) + ( builtins.tryEval (toInt " foo00123 ") == { success = false; value = false; } ) + ]; + # LISTS testFilter = { diff --git a/lib/tests/modules.sh b/lib/tests/modules.sh index 2be9b5835090..f6298297d136 100755 --- a/lib/tests/modules.sh +++ b/lib/tests/modules.sh @@ -162,7 +162,7 @@ checkConfigError 'A definition for option .* is not.*string or signed integer co # Check coerced value with unsound coercion checkConfigOutput '^12$' config.value ./declare-coerced-value-unsound.nix checkConfigError 'A definition for option .* is not of type .*. Definition values:\n\s*- In .*: "1000"' config.value ./declare-coerced-value-unsound.nix ./define-value-string-bigint.nix -checkConfigError 'json.exception.parse_error' config.value ./declare-coerced-value-unsound.nix ./define-value-string-arbitrary.nix +checkConfigError 'Could not convert .* to int.' config.value ./declare-coerced-value-unsound.nix ./define-value-string-arbitrary.nix # Check mkAliasOptionModule. checkConfigOutput '^true$' config.enable ./alias-with-priority.nix From 39a4ab78a1245eb45d333fc14ec56f3a8f045986 Mon Sep 17 00:00:00 2001 From: Jacob Abel Date: Fri, 8 Jul 2022 19:37:45 -0400 Subject: [PATCH 3/5] lib/strings: Refactor toInt into toInt and toIntBase10 --- lib/default.nix | 2 +- lib/strings.nix | 54 ++++++++++++++++++++++++++++++++++++--- lib/tests/misc.nix | 60 ++++++++++++++++++++++++++++++++------------ lib/tests/modules.sh | 2 +- 4 files changed, 97 insertions(+), 21 deletions(-) diff --git a/lib/default.nix b/lib/default.nix index 0c0e2d5e1021..8bb06954518b 100644 --- a/lib/default.nix +++ b/lib/default.nix @@ -103,7 +103,7 @@ let getName getVersion nameFromURL enableFeature enableFeatureAs withFeature withFeatureAs fixedWidthString fixedWidthNumber isStorePath - toInt readPathsFromFile fileContents; + toInt toIntBase10 readPathsFromFile fileContents; inherit (self.stringsWithDeps) textClosureList textClosureMap noDepEntry fullDepEntry packEntry stringAfter; inherit (self.customisation) overrideDerivation makeOverridable diff --git a/lib/strings.nix b/lib/strings.nix index c6269e755e2a..368ec786d670 100644 --- a/lib/strings.nix +++ b/lib/strings.nix @@ -783,26 +783,74 @@ rec { else false; - /* Parse a string as an int. + /* Parse a string as an int. Does not support parsing of integers with preceding zero due to + ambiguity between zero-padded and octal numbers. Type: string -> int Example: + toInt "1337" => 1337 + toInt "-4" => -4 + toInt " 123 " => 123 + toInt "00024" - => 24 + => error: [json.exception.parse_error.101] parse error at line 1, column 2: syntax error + while parsing value - unexpected number literal; expected end of input + toInt "3.14" => error: floating point JSON numbers are not supported */ # Obviously, it is a bit hacky to use fromJSON this way. toInt = str: let - # RegEx: Match any leading whitespace, then any zero padding, and capture any remaining + # RegEx: Match any leading whitespace, then any digits, and finally match any trailing + # whitespace. + strippedInput = match "[[:space:]]*([[:digit:]]+)[[:space:]]*" str; + + # RegEx: Match any leading whitespace, then a leading '0', then at least one digit following + # after, and finally match any trailing whitespace. + isLeadingZero = match "[[:space:]]*0[[:digit:]]+[[:space:]]*" str == []; + + # Attempt to parse input + parsedInput = fromJSON (elemAt strippedInput 0); + in + if isLeadingZero + then throw "Ambiguity in ${str} between octal and zero padded integer." + else if strippedInput != null && isInt parsedInput + then parsedInput + else throw "Could not convert ${str} to int."; + + + /* Parse a string as a base 10 int. This supports parsing of zero-padded integers. + + Type: string -> int + + Example: + toIntBase10 "1337" + => 1337 + + toIntBase10 "-4" + => -4 + + toIntBase10 " 123 " + => 123 + + toIntBase10 "00024" + => 24 + + toIntBase10 "3.14" + => error: floating point JSON numbers are not supported + */ + # Obviously, it is a bit hacky to use fromJSON this way. + toIntBase10 = str: + let + # RegEx: Match any leading whitespace, then match any zero padding, capture any remaining # digits after that, and finally match any trailing whitespace. strippedInput = match "[[:space:]]*0*([[:digit:]]+)[[:space:]]*" str; diff --git a/lib/tests/misc.nix b/lib/tests/misc.nix index 97d53026c644..4bfc8bb87699 100644 --- a/lib/tests/misc.nix +++ b/lib/tests/misc.nix @@ -339,20 +339,6 @@ runTests { (0 == toInt " 0") (0 == toInt "0 ") (0 == toInt " 0 ") - # Zero Padding - (123 == toInt "0123") - (123 == toInt "0000123") - (0 == toInt "000000") - # Whitespace and Zero Padding - (123 == toInt " 0123") - (123 == toInt "0123 ") - (123 == toInt " 0123 ") - (123 == toInt " 0000123") - (123 == toInt "0000123 ") - (123 == toInt " 0000123 ") - (0 == toInt " 000000") - (0 == toInt "000000 ") - (0 == toInt " 000000 ") ]; testToIntFails = testAllTrue [ @@ -360,10 +346,52 @@ runTests { ( builtins.tryEval (toInt "123 123") == { success = false; value = false; } ) ( builtins.tryEval (toInt "0 123") == { success = false; value = false; } ) ( builtins.tryEval (toInt " 0d ") == { success = false; value = false; } ) + ( builtins.tryEval (toInt "00") == { success = false; value = false; } ) + ( builtins.tryEval (toInt "01") == { success = false; value = false; } ) + ( builtins.tryEval (toInt "002") == { success = false; value = false; } ) + ( builtins.tryEval (toInt " 002 ") == { success = false; value = false; } ) ( builtins.tryEval (toInt " foo ") == { success = false; value = false; } ) ( builtins.tryEval (toInt " foo 123 ") == { success = false; value = false; } ) - ( builtins.tryEval (toInt " foo 00123 ") == { success = false; value = false; } ) - ( builtins.tryEval (toInt " foo00123 ") == { success = false; value = false; } ) + ( builtins.tryEval (toInt " foo123 ") == { success = false; value = false; } ) + ]; + + testToIntBase10 = testAllTrue [ + # Naive + (123 == toIntBase10 "123") + (0 == toIntBase10 "0") + # Whitespace Padding + (123 == toIntBase10 " 123") + (123 == toIntBase10 "123 ") + (123 == toIntBase10 " 123 ") + (123 == toIntBase10 " 123 ") + (0 == toIntBase10 " 0") + (0 == toIntBase10 "0 ") + (0 == toIntBase10 " 0 ") + # Zero Padding + (123 == toIntBase10 "0123") + (123 == toIntBase10 "0000123") + (0 == toIntBase10 "000000") + # Whitespace and Zero Padding + (123 == toIntBase10 " 0123") + (123 == toIntBase10 "0123 ") + (123 == toIntBase10 " 0123 ") + (123 == toIntBase10 " 0000123") + (123 == toIntBase10 "0000123 ") + (123 == toIntBase10 " 0000123 ") + (0 == toIntBase10 " 000000") + (0 == toIntBase10 "000000 ") + (0 == toIntBase10 " 000000 ") + ]; + + testToIntBase10Fails = testAllTrue [ + ( builtins.tryEval (toIntBase10 "") == { success = false; value = false; } ) + ( builtins.tryEval (toIntBase10 "123 123") == { success = false; value = false; } ) + ( builtins.tryEval (toIntBase10 "0 123") == { success = false; value = false; } ) + ( builtins.tryEval (toIntBase10 " 0d ") == { success = false; value = false; } ) + ( builtins.tryEval (toIntBase10 " foo ") == { success = false; value = false; } ) + ( builtins.tryEval (toIntBase10 " foo 123 ") == { success = false; value = false; } ) + ( builtins.tryEval (toIntBase10 " foo 00123 ") == { success = false; value = false; } ) + ( builtins.tryEval (toIntBase10 " foo00123 ") == { success = false; value = false; } ) ]; # LISTS diff --git a/lib/tests/modules.sh b/lib/tests/modules.sh index f6298297d136..92c28369ed5c 100755 --- a/lib/tests/modules.sh +++ b/lib/tests/modules.sh @@ -162,7 +162,7 @@ checkConfigError 'A definition for option .* is not.*string or signed integer co # Check coerced value with unsound coercion checkConfigOutput '^12$' config.value ./declare-coerced-value-unsound.nix checkConfigError 'A definition for option .* is not of type .*. Definition values:\n\s*- In .*: "1000"' config.value ./declare-coerced-value-unsound.nix ./define-value-string-bigint.nix -checkConfigError 'Could not convert .* to int.' config.value ./declare-coerced-value-unsound.nix ./define-value-string-arbitrary.nix +checkConfigError 'Could not convert .* to int' config.value ./declare-coerced-value-unsound.nix ./define-value-string-arbitrary.nix # Check mkAliasOptionModule. checkConfigOutput '^true$' config.enable ./alias-with-priority.nix From 88b18dcf445a1be963a6bd2f9e8c075edd668f71 Mon Sep 17 00:00:00 2001 From: Jacob Abel Date: Fri, 8 Jul 2022 20:17:44 -0400 Subject: [PATCH 4/5] lib/strings: Improve error message for octal ambiguity in toInt --- lib/strings.nix | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/lib/strings.nix b/lib/strings.nix index 368ec786d670..5c5507b597b6 100644 --- a/lib/strings.nix +++ b/lib/strings.nix @@ -800,8 +800,7 @@ rec { => 123 toInt "00024" - => error: [json.exception.parse_error.101] parse error at line 1, column 2: syntax error - while parsing value - unexpected number literal; expected end of input + => error: Ambiguity in interpretation of 00024 between octal and zero padded integer. toInt "3.14" => error: floating point JSON numbers are not supported @@ -821,7 +820,7 @@ rec { parsedInput = fromJSON (elemAt strippedInput 0); in if isLeadingZero - then throw "Ambiguity in ${str} between octal and zero padded integer." + then throw "Ambiguity in interpretation of ${str} between octal and zero padded integer." else if strippedInput != null && isInt parsedInput then parsedInput else throw "Could not convert ${str} to int."; From ed71173841618bd4c69f40d07fb467ccabc5db0b Mon Sep 17 00:00:00 2001 From: Jacob Abel Date: Sat, 9 Jul 2022 20:12:31 -0400 Subject: [PATCH 5/5] lib/strings: Update docs and restructured code to improve readability of toInt and toIntBase10. --- lib/strings.nix | 58 ++++++++++++++++++++++++++++---------------- lib/tests/misc.nix | 4 +++ lib/tests/modules.sh | 2 +- 3 files changed, 42 insertions(+), 22 deletions(-) diff --git a/lib/strings.nix b/lib/strings.nix index 5c5507b597b6..298d3b2af083 100644 --- a/lib/strings.nix +++ b/lib/strings.nix @@ -784,7 +784,7 @@ rec { false; /* Parse a string as an int. Does not support parsing of integers with preceding zero due to - ambiguity between zero-padded and octal numbers. + ambiguity between zero-padded and octal numbers. See toIntBase10. Type: string -> int @@ -805,25 +805,35 @@ rec { toInt "3.14" => error: floating point JSON numbers are not supported */ - # Obviously, it is a bit hacky to use fromJSON this way. toInt = str: let # RegEx: Match any leading whitespace, then any digits, and finally match any trailing # whitespace. strippedInput = match "[[:space:]]*([[:digit:]]+)[[:space:]]*" str; - # RegEx: Match any leading whitespace, then a leading '0', then at least one digit following - # after, and finally match any trailing whitespace. - isLeadingZero = match "[[:space:]]*0[[:digit:]]+[[:space:]]*" str == []; + # RegEx: Match a leading '0' then one or more digits. + isLeadingZero = match "0[[:digit:]]+" (head strippedInput) == []; # Attempt to parse input - parsedInput = fromJSON (elemAt strippedInput 0); + parsedInput = fromJSON (head strippedInput); + + generalError = "toInt: Could not convert ${escapeNixString str} to int."; + + octalAmbigError = "toInt: Ambiguity in interpretation of ${escapeNixString str}" + + " between octal and zero padded integer."; + in - if isLeadingZero - then throw "Ambiguity in interpretation of ${str} between octal and zero padded integer." - else if strippedInput != null && isInt parsedInput - then parsedInput - else throw "Could not convert ${str} to int."; + # Error on presence of non digit characters. + if strippedInput == null + then throw generalError + # Error on presence of leading zero/octal ambiguity. + else if isLeadingZero + then throw octalAmbigError + # Error if parse function fails. + else if !isInt parsedInput + then throw generalError + # Return result. + else parsedInput; /* Parse a string as a base 10 int. This supports parsing of zero-padded integers. @@ -846,26 +856,32 @@ rec { toIntBase10 "3.14" => error: floating point JSON numbers are not supported */ - # Obviously, it is a bit hacky to use fromJSON this way. toIntBase10 = str: let # RegEx: Match any leading whitespace, then match any zero padding, capture any remaining # digits after that, and finally match any trailing whitespace. strippedInput = match "[[:space:]]*0*([[:digit:]]+)[[:space:]]*" str; - # RegEx: Match any leading whitespace, at least one '0', and any trailing whitespace. - isZero = match "[[:space:]]*0+[[:space:]]*" str == []; + # RegEx: Match at least one '0'. + isZero = match "0+" (head strippedInput) == []; # Attempt to parse input - parsedInput = fromJSON (elemAt strippedInput 0); + parsedInput = fromJSON (head strippedInput); + + generalError = "toIntBase10: Could not convert ${escapeNixString str} to int."; + in - # Value is zero - if isZero + # Error on presence of non digit characters. + if strippedInput == null + then throw generalError + # In the special case zero-padded zero (00000), return early. + else if isZero then 0 - else - if strippedInput != null && isInt parsedInput - then parsedInput - else throw "Could not convert ${str} to int."; + # Error if parse function fails. + else if !isInt parsedInput + then throw generalError + # Return result. + else parsedInput; /* Read a list of paths from `file`, relative to the `rootPath`. Lines beginning with `#` are treated as comments and ignored. diff --git a/lib/tests/misc.nix b/lib/tests/misc.nix index 4bfc8bb87699..31c938a8ffda 100644 --- a/lib/tests/misc.nix +++ b/lib/tests/misc.nix @@ -346,6 +346,8 @@ runTests { ( builtins.tryEval (toInt "123 123") == { success = false; value = false; } ) ( builtins.tryEval (toInt "0 123") == { success = false; value = false; } ) ( builtins.tryEval (toInt " 0d ") == { success = false; value = false; } ) + ( builtins.tryEval (toInt " 1d ") == { success = false; value = false; } ) + ( builtins.tryEval (toInt " d0 ") == { success = false; value = false; } ) ( builtins.tryEval (toInt "00") == { success = false; value = false; } ) ( builtins.tryEval (toInt "01") == { success = false; value = false; } ) ( builtins.tryEval (toInt "002") == { success = false; value = false; } ) @@ -388,6 +390,8 @@ runTests { ( builtins.tryEval (toIntBase10 "123 123") == { success = false; value = false; } ) ( builtins.tryEval (toIntBase10 "0 123") == { success = false; value = false; } ) ( builtins.tryEval (toIntBase10 " 0d ") == { success = false; value = false; } ) + ( builtins.tryEval (toIntBase10 " 1d ") == { success = false; value = false; } ) + ( builtins.tryEval (toIntBase10 " d0 ") == { success = false; value = false; } ) ( builtins.tryEval (toIntBase10 " foo ") == { success = false; value = false; } ) ( builtins.tryEval (toIntBase10 " foo 123 ") == { success = false; value = false; } ) ( builtins.tryEval (toIntBase10 " foo 00123 ") == { success = false; value = false; } ) diff --git a/lib/tests/modules.sh b/lib/tests/modules.sh index 92c28369ed5c..c9ea674ee104 100755 --- a/lib/tests/modules.sh +++ b/lib/tests/modules.sh @@ -162,7 +162,7 @@ checkConfigError 'A definition for option .* is not.*string or signed integer co # Check coerced value with unsound coercion checkConfigOutput '^12$' config.value ./declare-coerced-value-unsound.nix checkConfigError 'A definition for option .* is not of type .*. Definition values:\n\s*- In .*: "1000"' config.value ./declare-coerced-value-unsound.nix ./define-value-string-bigint.nix -checkConfigError 'Could not convert .* to int' config.value ./declare-coerced-value-unsound.nix ./define-value-string-arbitrary.nix +checkConfigError 'toInt: Could not convert .* to int' config.value ./declare-coerced-value-unsound.nix ./define-value-string-arbitrary.nix # Check mkAliasOptionModule. checkConfigOutput '^true$' config.enable ./alias-with-priority.nix