gador/nixpkgs
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #197185 from Smona/handle-multiple-integrity-hashes

Browse Source 
yarn2nix: Handle lockfile entries with multiple integrity hashes
This commit is contained in:
Ryan Lahfa
2022-12-18 22:40:16 +01:00
committed by GitHub
parent f3aa5d8b67 1948179a74
commit 5b4485c8cd
11 changed files with 759 additions and 675 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkgs/development/tools/yarn2nix-moretea/yarn2nix/.prettierignore
+1
View File
@@ -0,0 +1 @@
node_modules
pkgs/development/tools/yarn2nix-moretea/yarn2nix/bin/yarn2nix.js
+31 -31
View File
@@ -1,14 +1,14 @@
#!/usr/bin/env node
const fs = require('fs')
const lockfile = require('@yarnpkg/lockfile')
const { docopt } = require('docopt')
const deepEqual = require('deep-equal')
const R = require('ramda')
const fs = require("fs");
const lockfile = require("@yarnpkg/lockfile");
const { docopt } = require("docopt");
const deepEqual = require("deep-equal");
const R = require("ramda");
const fixPkgAddMissingSha1 = require('../lib/fixPkgAddMissingSha1')
const mapObjIndexedReturnArray = require('../lib/mapObjIndexedReturnArray')
const generateNix = require('../lib/generateNix')
const fixPkgAddMissingSha1 = require("../lib/fixPkgAddMissingSha1");
const mapObjIndexedReturnArray = require("../lib/mapObjIndexedReturnArray");
const generateNix = require("../lib/generateNix");
const USAGE = `
Usage: yarn2nix [options]
@@ -19,11 +19,11 @@ Options:
--no-patch Don't patch the lockfile if hashes are missing
--lockfile=FILE Specify path to the lockfile [default: ./yarn.lock].
--builtin-fetchgit Use builtin fetchGit for git dependencies to support on-the-fly generation of yarn.nix without an internet connection
`
`;
const options = docopt(USAGE)
const options = docopt(USAGE);
const data = fs.readFileSync(options['--lockfile'], 'utf8')
const data = fs.readFileSync(options["--lockfile"], "utf8");
// json example:
@@ -45,10 +45,10 @@ const data = fs.readFileSync(options['--lockfile'], 'utf8')
// }
// }
const json = lockfile.parse(data)
const json = lockfile.parse(data);
if (json.type !== 'success') {
throw new Error('yarn.lock parse error')
if (json.type !== "success") {
throw new Error("yarn.lock parse error");
}
// Check for missing hashes in the yarn.lock and patch if necessary
@@ -56,35 +56,35 @@ if (json.type !== 'success') {
let pkgs = R.pipe(
mapObjIndexedReturnArray((value, key) => ({
...value,
nameWithVersion: key,
nameWithVersion: key
})),
R.uniqBy(R.prop('resolved')),
)(json.object)
R.uniqBy(R.prop("resolved"))
)(json.object);
;(async () => {
if (!options['--no-patch']) {
pkgs = await Promise.all(R.map(fixPkgAddMissingSha1, pkgs))
(async () => {
if (!options["--no-patch"]) {
pkgs = await Promise.all(R.map(fixPkgAddMissingSha1, pkgs));
}
const origJson = lockfile.parse(data)
const origJson = lockfile.parse(data);
if (!deepEqual(origJson, json)) {
console.error('found changes in the lockfile', options['--lockfile'])
console.error("found changes in the lockfile", options["--lockfile"]);
if (options['--no-patch']) {
console.error('...aborting')
process.exit(1)
if (options["--no-patch"]) {
console.error("...aborting");
process.exit(1);
}
fs.writeFileSync(options['--lockfile'], lockfile.stringify(json.object))
fs.writeFileSync(options["--lockfile"], lockfile.stringify(json.object));
}
if (!options['--no-nix']) {
if (!options["--no-nix"]) {
// print to stdout
console.log(generateNix(pkgs, options['--builtin-fetchgit']))
console.log(generateNix(pkgs, options["--builtin-fetchgit"]));
}
})().catch(error => {
console.error(error)
console.error(error);
process.exit(1)
})
process.exit(1);
});
pkgs/development/tools/yarn2nix-moretea/yarn2nix/internal/fixup_bin.js
+24 -24
View File
@@ -4,50 +4,50 @@
* node fixup_bin.js <bin_dir> <modules_dir> [<bin_pkg_1>, <bin_pkg_2> ... ]
*/
const fs = require('fs')
const path = require('path')
const fs = require("fs");
const path = require("path");
const derivationBinPath = process.argv[2]
const nodeModules = process.argv[3]
const packagesToPublishBin = process.argv.slice(4)
const derivationBinPath = process.argv[2];
const nodeModules = process.argv[3];
const packagesToPublishBin = process.argv.slice(4);
function processPackage(name) {
console.log('fixup_bin: Processing ', name)
console.log("fixup_bin: Processing ", name);
const packagePath = `${nodeModules}/${name}`
const packageJsonPath = `${packagePath}/package.json`
const packageJson = JSON.parse(fs.readFileSync(packageJsonPath))
const packagePath = `${nodeModules}/${name}`;
const packageJsonPath = `${packagePath}/package.json`;
const packageJson = JSON.parse(fs.readFileSync(packageJsonPath));
if (!packageJson.bin) {
console.log('fixup_bin: No binaries provided')
return
console.log("fixup_bin: No binaries provided");
return;
}
// There are two alternative syntaxes for `bin`
// a) just a plain string, in which case the name of the package is the name of the binary.
// b) an object, where key is the name of the eventual binary, and the value the path to that binary.
if (typeof packageJson.bin === 'string') {
const binName = packageJson.bin
packageJson.bin = {}
packageJson.bin[packageJson.name] = binName
if (typeof packageJson.bin === "string") {
const binName = packageJson.bin;
packageJson.bin = {};
packageJson.bin[packageJson.name] = binName;
}
// eslint-disable-next-line no-restricted-syntax, guard-for-in
for (const binName in packageJson.bin) {
const binPath = packageJson.bin[binName]
const normalizedBinName = binName.replace('@', '').replace('/', '-')
const binPath = packageJson.bin[binName];
const normalizedBinName = binName.replace("@", "").replace("/", "-");
const targetPath = path.normalize(`${packagePath}/${binPath}`)
const createdPath = `${derivationBinPath}/${normalizedBinName}`
const targetPath = path.normalize(`${packagePath}/${binPath}`);
const createdPath = `${derivationBinPath}/${normalizedBinName}`;
console.log(
`fixup_bin: creating link ${createdPath} that points to ${targetPath}`,
)
`fixup_bin: creating link ${createdPath} that points to ${targetPath}`
);
fs.symlinkSync(targetPath, createdPath)
fs.symlinkSync(targetPath, createdPath);
}
}
packagesToPublishBin.forEach(pkg => {
processPackage(pkg)
})
processPackage(pkg);
});
pkgs/development/tools/yarn2nix-moretea/yarn2nix/internal/fixup_yarn_lock.js
+21 -21
View File
@@ -4,46 +4,46 @@
* node fixup_yarn_lock.js yarn.lock
*/
const fs = require('fs')
const readline = require('readline')
const fs = require("fs");
const readline = require("readline");
const urlToName = require('../lib/urlToName')
const urlToName = require("../lib/urlToName");
const yarnLockPath = process.argv[2]
const yarnLockPath = process.argv[2];
const readFile = readline.createInterface({
input: fs.createReadStream(yarnLockPath, { encoding: 'utf8' }),
input: fs.createReadStream(yarnLockPath, { encoding: "utf8" }),
// Note: we use the crlfDelay option to recognize all instances of CR LF
// ('\r\n') in input.txt as a single line break.
crlfDelay: Infinity,
terminal: false, // input and output should be treated like a TTY
})
terminal: false // input and output should be treated like a TTY
});
const result = []
const result = [];
readFile
.on('line', line => {
const arr = line.match(/^ {2}resolved "([^#]+)(#[^"]+)?"$/)
.on("line", line => {
const arr = line.match(/^ {2}resolved "([^#]+)(#[^"]+)?"$/);
if (arr !== null) {
const [_, url, shaOrRev] = arr
const [_, url, shaOrRev] = arr;
const fileName = urlToName(url)
const fileName = urlToName(url);
result.push(` resolved "${fileName}${shaOrRev ?? ''}"`)
result.push(` resolved "${fileName}${shaOrRev ?? ""}"`);
} else {
result.push(line)
result.push(line);
}
})
.on('close', () => {
fs.writeFile(yarnLockPath, result.join('\n'), 'utf8', err => {
.on("close", () => {
fs.writeFile(yarnLockPath, result.join("\n"), "utf8", err => {
if (err) {
console.error(
'fixup_yarn_lock: fatal error when trying to write to yarn.lock',
err,
)
"fixup_yarn_lock: fatal error when trying to write to yarn.lock",
err
);
}
})
})
});
});
pkgs/development/tools/yarn2nix-moretea/yarn2nix/lib/fixPkgAddMissingSha1.js
+26 -28
View File
@@ -1,5 +1,5 @@
const https = require('https')
const crypto = require('crypto')
const https = require("https");
const crypto = require("crypto");
// TODO:
// make test case where getSha1 function is used, i.e. the case when resolved is without sha1?
@@ -8,29 +8,29 @@ const crypto = require('crypto')
function getSha1(url) {
return new Promise((resolve, reject) => {
https.get(url, res => {
const { statusCode } = res
const hash = crypto.createHash('sha1')
const { statusCode } = res;
const hash = crypto.createHash("sha1");
if (statusCode !== 200) {
const err = new Error(`Request Failed.\nStatus Code: ${statusCode}`)
const err = new Error(`Request Failed.\nStatus Code: ${statusCode}`);
// consume response data to free up memory
res.resume()
res.resume();
reject(err)
reject(err);
}
res.on('data', chunk => {
hash.update(chunk)
})
res.on("data", chunk => {
hash.update(chunk);
});
res.on('end', () => {
resolve(hash.digest('hex'))
})
res.on("end", () => {
resolve(hash.digest("hex"));
});
res.on('error', reject)
})
})
res.on("error", reject);
});
});
}
// Object -> Object
@@ -39,28 +39,26 @@ async function fixPkgAddMissingSha1(pkg) {
if (!pkg.resolved) {
console.error(
`yarn2nix: can't find "resolved" field for package ${
pkg.nameWithVersion
}, you probably required it using "file:...", this feature is not supported, ignoring`,
)
return pkg
`yarn2nix: can't find "resolved" field for package ${pkg.nameWithVersion}, you probably required it using "file:...", this feature is not supported, ignoring`
);
return pkg;
}
const [url, sha1] = pkg.resolved.split('#', 2)
const [url, sha1] = pkg.resolved.split("#", 2);
if (sha1 || url.startsWith('https://codeload.github.com')) {
return pkg
if (sha1 || url.startsWith("https://codeload.github.com")) {
return pkg;
}
// if there is no sha1 in resolved url
// (this could happen if yarn.lock was generated by older version of yarn)
// - request it from registry by https and add it to pkg
const newSha1 = await getSha1(url)
const newSha1 = await getSha1(url);
return {
...pkg,
resolved: `${url}#${newSha1}`,
}
resolved: `${url}#${newSha1}`
};
}
module.exports = fixPkgAddMissingSha1
module.exports = fixPkgAddMissingSha1;
pkgs/development/tools/yarn2nix-moretea/yarn2nix/lib/generateNix.js
+100 -58
View File
@@ -1,7 +1,8 @@
const R = require('ramda')
const R = require("ramda");
const ssri = require("ssri");
const urlToName = require('./urlToName')
const { execFileSync } = require('child_process')
const urlToName = require("./urlToName");
const { execFileSync } = require("child_process");
// fetchgit transforms
//
@@ -33,77 +34,115 @@ const { execFileSync } = require('child_process')
function prefetchgit(url, rev) {
return JSON.parse(
execFileSync("nix-prefetch-git", ["--rev", rev, url, "--fetch-submodules"], {
stdio: [ "ignore", "pipe", "ignore" ],
timeout: 60000,
})
).sha256
execFileSync(
"nix-prefetch-git",
["--rev", rev, url, "--fetch-submodules"],
{
stdio: ["ignore", "pipe", "ignore"],
timeout: 60000
}
)
).sha256;
}
function fetchgit(fileName, url, rev, branch, builtinFetchGit) {
const repo = builtinFetchGit
? `builtins.fetchGit ({
url = "${url}";
ref = "${branch}";
rev = "${rev}";
} // (if builtins.compareVersions "2.4pre" builtins.nixVersion < 0 then {
# workaround for https://github.com/NixOS/nix/issues/5128
allRefs = true;
} else {}))`
: `fetchgit {
url = "${url}";
rev = "${rev}";
sha256 = "${prefetchgit(url, rev)}";
}`;
return ` {
name = "${fileName}";
path =
let${builtinFetchGit ? `
repo = builtins.fetchGit ({
url = "${url}";
ref = "${branch}";
rev = "${rev}";
} // (if builtins.compareVersions "2.4pre" builtins.nixVersion < 0 then {
# workaround for https://github.com/NixOS/nix/issues/5128
allRefs = true;
} else {}));
` : `
repo = fetchgit {
url = "${url}";
rev = "${rev}";
sha256 = "${prefetchgit(url, rev)}";
};
`}in
runCommand "${fileName}" { buildInputs = [gnutar]; } ''
# Set u+w because tar-fs can't unpack archives with read-only dirs
# https://github.com/mafintosh/tar-fs/issues/79
tar cf $out --mode u+w -C \${repo} .
'';
}`
let repo = ${repo};
in runCommand "${fileName}" { buildInputs = [gnutar]; } ''
# Set u+w because tar-fs can't unpack archives with read-only dirs
# https://github.com/mafintosh/tar-fs/issues/79
tar cf $out --mode u+w -C \${repo} .
'';
}`;
}
/**
* Parse an integrity hash out of an SSRI string.
*
* Provides a default and uses the "best" supported algorithm if there are multiple.
*/
function parseIntegrity(maybeIntegrity, fallbackHash) {
if (!maybeIntegrity && fallbackHash) {
return { algo: "sha1", hash: fallbackHash };
}
const integrities = ssri.parse(maybeIntegrity);
for (const key in integrities) {
if (!/^sha(1|256|512)$/.test(key)) {
delete integrities[key];
}
}
algo = integrities.pickAlgorithm();
hash = integrities[algo][0].digest;
return { algo, hash };
}
function fetchLockedDep(builtinFetchGit) {
return function (pkg) {
const { integrity, nameWithVersion, resolved } = pkg
return function(pkg) {
const { integrity, nameWithVersion, resolved } = pkg;
if (!resolved) {
console.error(
`yarn2nix: can't find "resolved" field for package ${nameWithVersion}, you probably required it using "file:...", this feature is not supported, ignoring`,
)
return ''
`yarn2nix: can't find "resolved" field for package ${nameWithVersion}, you probably required it using "file:...", this feature is not supported, ignoring`
);
return "";
}
const [url, sha1OrRev] = resolved.split('#')
const [url, sha1OrRev] = resolved.split("#");
const fileName = urlToName(url)
const fileName = urlToName(url);
if (resolved.startsWith('https://codeload.github.com/')) {
const s = resolved.split('/')
const githubUrl = `https://github.com/${s[3]}/${s[4]}.git`
const githubRev = s[6]
if (resolved.startsWith("https://codeload.github.com/")) {
const s = resolved.split("/");
const githubUrl = `https://github.com/${s[3]}/${s[4]}.git`;
const githubRev = s[6];
const [_, branch] = nameWithVersion.split('#')
const [_, branch] = nameWithVersion.split("#");
return fetchgit(fileName, githubUrl, githubRev, branch || 'master', builtinFetchGit)
return fetchgit(
fileName,
githubUrl,
githubRev,
branch || "master",
builtinFetchGit
);
}
if (url.startsWith('git+') || url.startsWith("git:")) {
const rev = sha1OrRev
if (url.startsWith("git+") || url.startsWith("git:")) {
const rev = sha1OrRev;
const [_, branch] = nameWithVersion.split('#')
const [_, branch] = nameWithVersion.split("#");
const urlForGit = url.replace(/^git\+/, '')
const urlForGit = url.replace(/^git\+/, "");
return fetchgit(fileName, urlForGit, rev, branch || 'master', builtinFetchGit)
return fetchgit(
fileName,
urlForGit,
rev,
branch || "master",
builtinFetchGit
);
}
const [algo, hash] = integrity ? integrity.split('-') : ['sha1', sha1OrRev]
const { algo, hash } = parseIntegrity(integrity, sha1OrRev);
return ` {
name = "${fileName}";
@@ -112,26 +151,29 @@ function fetchLockedDep(builtinFetchGit) {
url = "${url}";
${algo} = "${hash}";
};
}`
}
}`;
};
}
const HEAD = `
{ fetchurl, fetchgit, linkFarm, runCommand, gnutar }: rec {
offline_cache = linkFarm "offline" packages;
packages = [
`.trim()
`.trim();
// Object -> String
function generateNix(pkgs, builtinFetchGit) {
const nameWithVersionAndPackageNix = R.map(fetchLockedDep(builtinFetchGit), pkgs)
const nameWithVersionAndPackageNix = R.map(
fetchLockedDep(builtinFetchGit),
pkgs
);
const packagesDefinition = R.join(
'\n',
R.values(nameWithVersionAndPackageNix),
)
"\n",
R.values(nameWithVersionAndPackageNix)
);
return R.join('\n', [HEAD, packagesDefinition, ' ];', '}'])
return R.join("\n", [HEAD, packagesDefinition, " ];", "}"]);
}
module.exports = generateNix
module.exports = generateNix;
pkgs/development/tools/yarn2nix-moretea/yarn2nix/lib/mapObjIndexedReturnArray.js
+6 -6
View File
@@ -1,6 +1,6 @@
const _curry2 = require('ramda/src/internal/_curry2')
const _map = require('ramda/src/internal/_map')
const keys = require('ramda/src/keys')
const _curry2 = require("ramda/src/internal/_curry2");
const _map = require("ramda/src/internal/_map");
const keys = require("ramda/src/keys");
// mapObjIndexed: ((v, k, {k: v}) → v') → {k: v} → {k: v'}
// mapObjIndexedReturnArray: ((v, k, {k: v}) → v') → {k: v} → [v']
@@ -15,7 +15,7 @@ const keys = require('ramda/src/keys')
*/
const mapObjIndexedReturnArray = _curry2((fn, obj) =>
_map(key => fn(obj[key], key, obj), keys(obj)),
)
_map(key => fn(obj[key], key, obj), keys(obj))
);
module.exports = mapObjIndexedReturnArray
module.exports = mapObjIndexedReturnArray;
pkgs/development/tools/yarn2nix-moretea/yarn2nix/lib/urlToName.js
+7 -8
View File
@@ -1,4 +1,4 @@
const path = require('path')
const path = require("path");
// String -> String
@@ -10,20 +10,19 @@ const path = require('path')
// - https://codeload.github.com/Gargron/emoji-mart/tar.gz/934f314fd8322276765066e8a2a6be5bac61b1cf
function urlToName(url) {
// Yarn generates `codeload.github.com` tarball URLs, where the final
// path component (file name) is the git hash. See #111.
// See also https://github.com/yarnpkg/yarn/blob/989a7406/src/resolvers/exotics/github-resolver.js#L24-L26
let isCodeloadGitTarballUrl =
url.startsWith('https://codeload.github.com/') && url.includes('/tar.gz/')
url.startsWith("https://codeload.github.com/") && url.includes("/tar.gz/");
if (url.startsWith('git+') || isCodeloadGitTarballUrl) {
return path.basename(url)
if (url.startsWith("git+") || isCodeloadGitTarballUrl) {
return path.basename(url);
}
return url
.replace(/https:\/\/(.)*(.com)\//g, '') // prevents having long directory names
.replace(/[@/%:-]/g, '_') // replace @ and : and - and % characters with underscore
.replace(/https:\/\/(.)*(.com)\//g, "") // prevents having long directory names
.replace(/[@/%:-]/g, "_"); // replace @ and : and - and % characters with underscore
}
module.exports = urlToName
module.exports = urlToName;
pkgs/development/tools/yarn2nix-moretea/yarn2nix/package.json
+4 -3
View File
@@ -14,14 +14,15 @@
"bin": {
"yarn2nix": "bin/yarn2nix.js"
},
"engines" : {
"node" : ">=8.0.0"
"engines": {
"node": ">=8.0.0"
},
"dependencies": {
"@yarnpkg/lockfile": "^1.1.0",
"deep-equal": "^1.0.1",
"docopt": "^0.6.2",
"ramda": "^0.26.1"
"ramda": "^0.26.1",
"ssri": "^10.0.0"
},
"devDependencies": {
"babel-eslint": "^10.0.1",
pkgs/development/tools/yarn2nix-moretea/yarn2nix/yarn.lock
+19
View File
@@ -2228,6 +2228,13 @@ minimist@^1.2.0, minimist@^1.2.5:
resolved "https://registry.yarnpkg.com/minimist/-/minimist-1.2.5.tgz#67d66014b66a6a8aaa0c083c5fd58df4e4e97602"
integrity sha512-FM9nNUYrRBAELZQT3xeZQ7fmMOBg6nWNmJKTcgsJeaLstP/UODVpGsr5OhXhhXg6f+qtJ8uiZ+PUxkDWcgIXLw==
minipass@^3.1.1:
version "3.3.4"
resolved "https://registry.yarnpkg.com/minipass/-/minipass-3.3.4.tgz#ca99f95dd77c43c7a76bf51e6d200025eee0ffae"
integrity sha512-I9WPbWHCGu8W+6k1ZiGpPu0GkoKBeorkfKNuAFBNS1HNFJvke82sxvI5bzcCNpWPorkOO5QQ+zomzzwRxejXiw==
dependencies:
yallist "^4.0.0"
mixin-deep@^1.2.0:
version "1.3.2"
resolved "https://registry.yarnpkg.com/mixin-deep/-/mixin-deep-1.3.2.tgz#1120b43dc359a785dce65b55b82e257ccf479566"
@@ -3136,6 +3143,13 @@ sprintf-js@~1.0.2:
resolved "https://registry.yarnpkg.com/sprintf-js/-/sprintf-js-1.0.3.tgz#04e6926f662895354f3dd015203633b857297e2c"
integrity sha1-BOaSb2YolTVPPdAVIDYzuFcpfiw=
ssri@^10.0.0:
version "10.0.0"
resolved "https://registry.yarnpkg.com/ssri/-/ssri-10.0.0.tgz#1e34554cbbc4728f5290674264e21b64aaf27ca7"
integrity sha512-64ghGOpqW0k+jh7m5jndBGdVEoPikWwGQmBNN5ks6jyUSMymzHDTlnNHOvzp+6MmHOljr2MokUzvRksnTwG0Iw==
dependencies:
minipass "^3.1.1"
staged-git-files@1.1.2:
version "1.1.2"
resolved "https://registry.yarnpkg.com/staged-git-files/-/staged-git-files-1.1.2.tgz#4326d33886dc9ecfa29a6193bf511ba90a46454b"
@@ -3545,6 +3559,11 @@ yallist@^2.1.2:
resolved "https://registry.yarnpkg.com/yallist/-/yallist-2.1.2.tgz#1c11f9218f076089a47dd512f93c6699a6a81d52"
integrity sha1-HBH5IY8HYImkfdUS+TxmmaaoHVI=
yallist@^4.0.0:
version "4.0.0"
resolved "https://registry.yarnpkg.com/yallist/-/yallist-4.0.0.tgz#9bb92790d9c0effec63be73519e11a35019a3a72"
integrity sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==
yargs-parser@^8.0.0:
version "8.1.0"
resolved "https://registry.yarnpkg.com/yargs-parser/-/yargs-parser-8.1.0.tgz#f1376a33b6629a5d063782944da732631e966950"
pkgs/development/tools/yarn2nix-moretea/yarn2nix/yarn.nix
+520 -496
View File
File diff suppressed because it is too large Load Diff