fetchers: add fetchFromHuggingFace
Add a Hugging Face Hub repository fetcher with support for model, dataset, and space repositories, custom Hub domains, tags and revisions, and the relevant fetchgit options. Select storage through a backend argument. Git LFS is implemented through fetchgit; Xet is the default to match Hugging Face but fails explicitly until its implementation lands. Expose the helper from pkgs, document its API, and register evaluation and fixed-output fetch tests. Assisted-by: OpenAI Codex (GPT-5)
This commit is contained in:
Hugo Herter
@@ -853,7 +853,7 @@ Used with CVS. Expects `cvsRoot`, `tag`, and `hash`.
|
||||
|
||||
Used with Mercurial. Expects `url`, `rev`, `hash`, overridable with [`<pkg>.overrideAttrs`](#sec-pkg-overrideAttrs).
|
||||
|
||||
A number of fetcher functions wrap part of `fetchurl` and `fetchzip`. They are mainly convenience functions intended for commonly used destinations of source code in Nixpkgs. These wrapper fetchers are listed below.
|
||||
A number of fetcher functions wrap lower-level fetchers such as `fetchurl`, `fetchzip`, and `fetchgit`. They are mainly convenience functions intended for commonly used destinations of source code in Nixpkgs. These wrapper fetchers are listed below.
|
||||
|
||||
## `fetchFromGitea`, `fetchFromForgejo` and `fetchFromCodeberg` {#fetchfromgitea}
|
||||
|
||||
@@ -876,6 +876,45 @@ However, `fetchFromGitHub` will automatically switch to using `fetchgit` in any
|
||||
|
||||
When `fetchgit` is used, refer to the `fetchgit` section for documentation of its available options.
|
||||
|
||||
## `fetchFromHuggingFace` {#fetchfromhuggingface}
|
||||
|
||||
`fetchFromHuggingFace` fetches repositories from Hugging Face Hub. It expects
|
||||
`repoId`, exactly one of `rev` or `tag`, and `hash`.
|
||||
|
||||
`repoId` must be in the form `repo` or `owner/repo`, so repositories such as
|
||||
`gpt2` work as well.
|
||||
|
||||
::: {.example #ex-fetchfromhuggingface}
|
||||
|
||||
# Fetching a model repository from Hugging Face
|
||||
|
||||
```nix
|
||||
fetchFromHuggingFace {
|
||||
repoId = "hf-internal-testing/tiny-random-gpt2";
|
||||
rev = "71034c5d8bde858ff824298bdedc65515b97d2b9";
|
||||
backend = "lfs";
|
||||
hash = "sha256-8K9B/C62GW5lXC0c8QQpQ9QAE1UMoG+kYqvGhnWIp64=";
|
||||
}
|
||||
```
|
||||
|
||||
:::
|
||||
|
||||
The optional `repoType` argument selects which Hugging Face Hub repository type
|
||||
to use:
|
||||
|
||||
- `"model"` (default) fetches from `https://huggingface.co/<repo-id>`
|
||||
- `"dataset"` fetches from `https://huggingface.co/datasets/<repo-id>`
|
||||
- `"space"` fetches from `https://huggingface.co/spaces/<repo-id>`
|
||||
|
||||
To use a different Hugging Face Hub instance, use `domain`
|
||||
(defaults to `"huggingface.co"`).
|
||||
|
||||
The optional `backend` argument defaults to `"xet"`. Because the Xet backend is
|
||||
not implemented yet, callers must currently set `backend = "lfs"`, which uses
|
||||
`fetchgit` with Git LFS enabled and defaults `fetchSubmodules` to `false`.
|
||||
`rootDir`, `sparseCheckout`, and low-level `fetchgit` options such as
|
||||
`deepClone`, `fetchTags`, `leaveDotGit`, and `branchName` are also supported.
|
||||
|
||||
## `fetchFromGitLab` {#fetchfromgitlab}
|
||||
|
||||
This is used with GitLab repositories. It behaves similarly to `fetchFromGitHub`, and expects `owner`, `repo`, `rev`, and `hash`.
|
||||
|
||||
@@ -2061,6 +2061,12 @@
|
||||
"fetchfromgithub": [
|
||||
"index.html#fetchfromgithub"
|
||||
],
|
||||
"fetchfromhuggingface": [
|
||||
"index.html#fetchfromhuggingface"
|
||||
],
|
||||
"ex-fetchfromhuggingface": [
|
||||
"index.html#ex-fetchfromhuggingface"
|
||||
],
|
||||
"fetchfromgitlab": [
|
||||
"index.html#fetchfromgitlab"
|
||||
],
|
||||
|
||||
@@ -0,0 +1,115 @@
|
||||
{
|
||||
lib,
|
||||
repoRevToNameMaybe,
|
||||
fetchgit,
|
||||
}:
|
||||
|
||||
let
|
||||
repoPrefixes = {
|
||||
model = "";
|
||||
dataset = "datasets/";
|
||||
space = "spaces/";
|
||||
};
|
||||
in
|
||||
|
||||
lib.makeOverridable (
|
||||
{
|
||||
repoId,
|
||||
tag ? null,
|
||||
rev ? null,
|
||||
name ? repoRevToNameMaybe repoId (lib.revOrTag rev tag) "huggingface",
|
||||
domain ? "huggingface.co",
|
||||
repoType ? "model",
|
||||
backend ? "xet",
|
||||
branchName ? null,
|
||||
deepClone ? false,
|
||||
fetchSubmodules ? false,
|
||||
fetchTags ? false,
|
||||
leaveDotGit ? null,
|
||||
rootDir ? "",
|
||||
sparseCheckout ? null,
|
||||
passthru ? { },
|
||||
meta ? { },
|
||||
... # For hash agility and additional fetchgit arguments
|
||||
}@args:
|
||||
|
||||
assert (
|
||||
lib.assertMsg (lib.xor (tag == null) (
|
||||
rev == null
|
||||
)) "fetchFromHuggingFace requires one of either `rev` or `tag` to be provided (not both)."
|
||||
);
|
||||
|
||||
assert (lib.assertOneOf "repoType" repoType (builtins.attrNames repoPrefixes));
|
||||
assert (
|
||||
lib.assertOneOf "backend" backend [
|
||||
"lfs"
|
||||
"xet"
|
||||
]
|
||||
);
|
||||
|
||||
let
|
||||
position = (
|
||||
if args.meta.description or null != null then
|
||||
builtins.unsafeGetAttrPos "description" args.meta
|
||||
else if tag != null then
|
||||
builtins.unsafeGetAttrPos "tag" args
|
||||
else
|
||||
builtins.unsafeGetAttrPos "rev" args
|
||||
);
|
||||
baseUrl = "https://${domain}/${repoPrefixes.${repoType}}${repoId}";
|
||||
gitRepoUrl = "${baseUrl}.git";
|
||||
newMeta =
|
||||
meta
|
||||
// {
|
||||
homepage = meta.homepage or baseUrl;
|
||||
}
|
||||
// lib.optionalAttrs (position != null) {
|
||||
# to indicate where derivation originates, similar to make-derivation.nix's mkDerivation
|
||||
position = "${position.file}:${toString position.line}";
|
||||
};
|
||||
backendFetcher = builtins.getAttr backend {
|
||||
lfs = fetchgit;
|
||||
xet = throw "fetchFromHuggingFace: the Xet backend is not implemented yet";
|
||||
};
|
||||
in
|
||||
assert (
|
||||
lib.assertMsg (
|
||||
builtins.match "[^/]+(/[^/]+)?" repoId != null
|
||||
) "fetchFromHuggingFace requires `repoId` to be in the form `repo` or `owner/repo`."
|
||||
);
|
||||
backendFetcher (
|
||||
removeAttrs args [
|
||||
"backend"
|
||||
"domain"
|
||||
"repoId"
|
||||
"repoType"
|
||||
]
|
||||
// {
|
||||
inherit
|
||||
branchName
|
||||
deepClone
|
||||
fetchSubmodules
|
||||
fetchTags
|
||||
leaveDotGit
|
||||
name
|
||||
rootDir
|
||||
sparseCheckout
|
||||
tag
|
||||
rev
|
||||
;
|
||||
url = gitRepoUrl;
|
||||
fetchLFS = true;
|
||||
meta = newMeta;
|
||||
passthru = {
|
||||
inherit gitRepoUrl;
|
||||
}
|
||||
// passthru;
|
||||
}
|
||||
)
|
||||
// {
|
||||
inherit
|
||||
repoId
|
||||
repoType
|
||||
;
|
||||
}
|
||||
)
|
||||
@@ -0,0 +1,163 @@
|
||||
{
|
||||
lib,
|
||||
testers,
|
||||
fetchFromHuggingFace,
|
||||
runCommand,
|
||||
...
|
||||
}:
|
||||
let
|
||||
fetchWithLFS = args: fetchFromHuggingFace (args // { backend = "lfs"; });
|
||||
fetchTestRepository = testers.invalidateFetcherByDrvHash fetchWithLFS;
|
||||
fakeRev = "0123456789abcdef0123456789abcdef01234567";
|
||||
|
||||
expectEvalFailure =
|
||||
name: expr:
|
||||
let
|
||||
result = builtins.tryEval expr;
|
||||
in
|
||||
runCommand "${name}-test" { } ''
|
||||
test "${if result.success then "1" else "0"}" = "0"
|
||||
touch "$out"
|
||||
'';
|
||||
in
|
||||
{
|
||||
apiSurface =
|
||||
let
|
||||
unnamespaced = fetchWithLFS {
|
||||
repoId = "gpt2";
|
||||
rev = fakeRev;
|
||||
hash = lib.fakeHash;
|
||||
};
|
||||
tagged = fetchWithLFS {
|
||||
repoId = "kitten/tagged-model";
|
||||
tag = "v1.0";
|
||||
hash = lib.fakeHash;
|
||||
};
|
||||
dataset = fetchWithLFS {
|
||||
repoId = "kitten/dataset";
|
||||
repoType = "dataset";
|
||||
domain = "hf.example";
|
||||
rev = fakeRev;
|
||||
hash = lib.fakeHash;
|
||||
};
|
||||
space = fetchWithLFS {
|
||||
repoId = "kitten/space";
|
||||
repoType = "space";
|
||||
rev = fakeRev;
|
||||
hash = lib.fakeHash;
|
||||
passthru.custom = "value";
|
||||
};
|
||||
fetchgitOptions = fetchWithLFS {
|
||||
repoId = "kitten/fetchgit-options";
|
||||
rev = fakeRev;
|
||||
branchName = "huggingface";
|
||||
deepClone = true;
|
||||
fetchSubmodules = true;
|
||||
fetchTags = true;
|
||||
leaveDotGit = true;
|
||||
hash = lib.fakeHash;
|
||||
};
|
||||
in
|
||||
runCommand "fetchFromHuggingFace-api-surface-test" { } ''
|
||||
test "${unnamespaced.repoId}" = "gpt2"
|
||||
test "${unnamespaced.repoType}" = "model"
|
||||
test "${unnamespaced.passthru.gitRepoUrl}" = "https://huggingface.co/gpt2.git"
|
||||
test "${unnamespaced.meta.homepage}" = "https://huggingface.co/gpt2"
|
||||
test "${if unnamespaced.fetchLFS then "1" else "0"}" = "1"
|
||||
|
||||
test "${tagged.tag}" = "v1.0"
|
||||
test "${tagged.rev}" = "refs/tags/v1.0"
|
||||
test "${tagged.passthru.gitRepoUrl}" = "https://huggingface.co/kitten/tagged-model.git"
|
||||
|
||||
test "${dataset.repoId}" = "kitten/dataset"
|
||||
test "${dataset.repoType}" = "dataset"
|
||||
test "${dataset.passthru.gitRepoUrl}" = "https://hf.example/datasets/kitten/dataset.git"
|
||||
test "${dataset.meta.homepage}" = "https://hf.example/datasets/kitten/dataset"
|
||||
|
||||
test "${space.repoType}" = "space"
|
||||
test "${space.passthru.gitRepoUrl}" = "https://huggingface.co/spaces/kitten/space.git"
|
||||
test "${if space.fetchLFS then "1" else "0"}" = "1"
|
||||
test "${space.passthru.custom}" = "value"
|
||||
|
||||
test "${fetchgitOptions.branchName}" = "huggingface"
|
||||
test "${if fetchgitOptions.deepClone then "1" else "0"}" = "1"
|
||||
test "${if fetchgitOptions.fetchLFS then "1" else "0"}" = "1"
|
||||
test "${if fetchgitOptions.fetchSubmodules then "1" else "0"}" = "1"
|
||||
test "${if fetchgitOptions.fetchTags then "1" else "0"}" = "1"
|
||||
test "${if fetchgitOptions.leaveDotGit then "1" else "0"}" = "1"
|
||||
|
||||
touch "$out"
|
||||
'';
|
||||
|
||||
missingRevOrTag = expectEvalFailure "fetchFromHuggingFace-missing-rev-or-tag" (
|
||||
(fetchFromHuggingFace {
|
||||
repoId = "gpt2";
|
||||
hash = lib.fakeHash;
|
||||
}).drvPath
|
||||
);
|
||||
|
||||
bothRevAndTag = expectEvalFailure "fetchFromHuggingFace-both-rev-and-tag" (
|
||||
(fetchFromHuggingFace {
|
||||
repoId = "gpt2";
|
||||
rev = fakeRev;
|
||||
tag = "main";
|
||||
hash = lib.fakeHash;
|
||||
}).drvPath
|
||||
);
|
||||
|
||||
invalidRepoId = expectEvalFailure "fetchFromHuggingFace-invalid-repo-id" (
|
||||
(fetchFromHuggingFace {
|
||||
repoId = "broken/repo/id";
|
||||
rev = fakeRev;
|
||||
hash = lib.fakeHash;
|
||||
}).drvPath
|
||||
);
|
||||
|
||||
invalidRepoType = expectEvalFailure "fetchFromHuggingFace-invalid-repo-type" (
|
||||
(fetchFromHuggingFace {
|
||||
repoId = "gpt2";
|
||||
repoType = "collection";
|
||||
rev = fakeRev;
|
||||
hash = lib.fakeHash;
|
||||
}).drvPath
|
||||
);
|
||||
|
||||
defaultXetBackend = expectEvalFailure "fetchFromHuggingFace-default-xet-backend" (
|
||||
(fetchFromHuggingFace {
|
||||
repoId = "gpt2";
|
||||
rev = fakeRev;
|
||||
hash = lib.fakeHash;
|
||||
}).drvPath
|
||||
);
|
||||
|
||||
explicitXetBackend = expectEvalFailure "fetchFromHuggingFace-explicit-xet-backend" (
|
||||
(fetchFromHuggingFace {
|
||||
repoId = "gpt2";
|
||||
rev = fakeRev;
|
||||
backend = "xet";
|
||||
hash = lib.fakeHash;
|
||||
}).drvPath
|
||||
);
|
||||
|
||||
invalidBackend = expectEvalFailure "fetchFromHuggingFace-invalid-backend" (
|
||||
(fetchFromHuggingFace {
|
||||
repoId = "gpt2";
|
||||
rev = fakeRev;
|
||||
backend = "git";
|
||||
hash = lib.fakeHash;
|
||||
}).drvPath
|
||||
);
|
||||
|
||||
simple = fetchTestRepository {
|
||||
repoId = "hf-internal-testing/tiny-random-gpt2";
|
||||
rev = "71034c5d8bde858ff824298bdedc65515b97d2b9";
|
||||
hash = "sha256-8K9B/C62GW5lXC0c8QQpQ9QAE1UMoG+kYqvGhnWIp64=";
|
||||
};
|
||||
|
||||
rootDir = fetchTestRepository {
|
||||
repoId = "hf-internal-testing/tiny-random-BertModel";
|
||||
rev = "fc08ad9cc33be9aef4f55cc80e16ef5ae3d5981c";
|
||||
rootDir = "onnx";
|
||||
hash = "sha256-ETm2DT9jvVJ5W3MP8T0RiulNUlXlA2chtc9AVI+u6n4=";
|
||||
};
|
||||
}
|
||||
@@ -153,6 +153,9 @@ in
|
||||
);
|
||||
fetchFromBitbucket = recurseIntoAttrs (callPackages ../build-support/fetchbitbucket/tests.nix { });
|
||||
fetchFromGitHub = recurseIntoAttrs (callPackages ../build-support/fetchgithub/tests.nix { });
|
||||
fetchFromHuggingFace = recurseIntoAttrs (
|
||||
callPackages ../build-support/fetchhuggingface/tests.nix { }
|
||||
);
|
||||
fetchFirefoxAddon = recurseIntoAttrs (
|
||||
callPackages ../build-support/fetchfirefoxaddon/tests.nix { }
|
||||
);
|
||||
|
||||
@@ -648,6 +648,8 @@ with pkgs;
|
||||
|
||||
fetchFromGitHub = callPackage ../build-support/fetchgithub { };
|
||||
|
||||
fetchFromHuggingFace = callPackage ../build-support/fetchhuggingface { };
|
||||
|
||||
fetchFromBitbucket = callPackage ../build-support/fetchbitbucket { };
|
||||
|
||||
fetchFromSavannah = callPackage ../build-support/fetchsavannah { };
|
||||
|
||||
Reference in New Issue
Block a user