diff --git a/doc/build-helpers/fetchers.chapter.md b/doc/build-helpers/fetchers.chapter.md index 894d52c4b6b6..edb9ab4e0243 100644 --- a/doc/build-helpers/fetchers.chapter.md +++ b/doc/build-helpers/fetchers.chapter.md @@ -853,7 +853,7 @@ Used with CVS. Expects `cvsRoot`, `tag`, and `hash`. Used with Mercurial. Expects `url`, `rev`, `hash`, overridable with [`.overrideAttrs`](#sec-pkg-overrideAttrs). -A number of fetcher functions wrap part of `fetchurl` and `fetchzip`. They are mainly convenience functions intended for commonly used destinations of source code in Nixpkgs. These wrapper fetchers are listed below. +A number of fetcher functions wrap lower-level fetchers such as `fetchurl`, `fetchzip`, and `fetchgit`. They are mainly convenience functions intended for commonly used destinations of source code in Nixpkgs. These wrapper fetchers are listed below. ## `fetchFromGitea`, `fetchFromForgejo` and `fetchFromCodeberg` {#fetchfromgitea} @@ -876,6 +876,45 @@ However, `fetchFromGitHub` will automatically switch to using `fetchgit` in any When `fetchgit` is used, refer to the `fetchgit` section for documentation of its available options. +## `fetchFromHuggingFace` {#fetchfromhuggingface} + +`fetchFromHuggingFace` fetches repositories from Hugging Face Hub. It expects +`repoId`, exactly one of `rev` or `tag`, and `hash`. + +`repoId` must be in the form `repo` or `owner/repo`, so repositories such as +`gpt2` work as well. + +::: {.example #ex-fetchfromhuggingface} + +# Fetching a model repository from Hugging Face + +```nix +fetchFromHuggingFace { + repoId = "hf-internal-testing/tiny-random-gpt2"; + rev = "71034c5d8bde858ff824298bdedc65515b97d2b9"; + backend = "lfs"; + hash = "sha256-8K9B/C62GW5lXC0c8QQpQ9QAE1UMoG+kYqvGhnWIp64="; +} +``` + +::: + +The optional `repoType` argument selects which Hugging Face Hub repository type +to use: + +- `"model"` (default) fetches from `https://huggingface.co/` +- `"dataset"` fetches from `https://huggingface.co/datasets/` +- `"space"` fetches from `https://huggingface.co/spaces/` + +To use a different Hugging Face Hub instance, use `domain` +(defaults to `"huggingface.co"`). + +The optional `backend` argument defaults to `"xet"`. Because the Xet backend is +not implemented yet, callers must currently set `backend = "lfs"`, which uses +`fetchgit` with Git LFS enabled and defaults `fetchSubmodules` to `false`. +`rootDir`, `sparseCheckout`, and low-level `fetchgit` options such as +`deepClone`, `fetchTags`, `leaveDotGit`, and `branchName` are also supported. + ## `fetchFromGitLab` {#fetchfromgitlab} This is used with GitLab repositories. It behaves similarly to `fetchFromGitHub`, and expects `owner`, `repo`, `rev`, and `hash`. diff --git a/doc/redirects.json b/doc/redirects.json index 19ac1ce89977..08c6beac3383 100644 --- a/doc/redirects.json +++ b/doc/redirects.json @@ -2062,6 +2062,12 @@ "fetchfromgithub": [ "index.html#fetchfromgithub" ], + "fetchfromhuggingface": [ + "index.html#fetchfromhuggingface" + ], + "ex-fetchfromhuggingface": [ + "index.html#ex-fetchfromhuggingface" + ], "fetchfromgitlab": [ "index.html#fetchfromgitlab" ], diff --git a/pkgs/build-support/fetchhuggingface/default.nix b/pkgs/build-support/fetchhuggingface/default.nix new file mode 100644 index 000000000000..9bf33b248655 --- /dev/null +++ b/pkgs/build-support/fetchhuggingface/default.nix @@ -0,0 +1,115 @@ +{ + lib, + repoRevToNameMaybe, + fetchgit, +}: + +let + repoPrefixes = { + model = ""; + dataset = "datasets/"; + space = "spaces/"; + }; +in + +lib.makeOverridable ( + { + repoId, + tag ? null, + rev ? null, + name ? repoRevToNameMaybe repoId (lib.revOrTag rev tag) "huggingface", + domain ? "huggingface.co", + repoType ? "model", + backend ? "xet", + branchName ? null, + deepClone ? false, + fetchSubmodules ? false, + fetchTags ? false, + leaveDotGit ? null, + rootDir ? "", + sparseCheckout ? null, + passthru ? { }, + meta ? { }, + ... # For hash agility and additional fetchgit arguments + }@args: + + assert ( + lib.assertMsg (lib.xor (tag == null) ( + rev == null + )) "fetchFromHuggingFace requires one of either `rev` or `tag` to be provided (not both)." + ); + + assert (lib.assertOneOf "repoType" repoType (builtins.attrNames repoPrefixes)); + assert ( + lib.assertOneOf "backend" backend [ + "lfs" + "xet" + ] + ); + + let + position = ( + if args.meta.description or null != null then + builtins.unsafeGetAttrPos "description" args.meta + else if tag != null then + builtins.unsafeGetAttrPos "tag" args + else + builtins.unsafeGetAttrPos "rev" args + ); + baseUrl = "https://${domain}/${repoPrefixes.${repoType}}${repoId}"; + gitRepoUrl = "${baseUrl}.git"; + newMeta = + meta + // { + homepage = meta.homepage or baseUrl; + } + // lib.optionalAttrs (position != null) { + # to indicate where derivation originates, similar to make-derivation.nix's mkDerivation + position = "${position.file}:${toString position.line}"; + }; + backendFetcher = builtins.getAttr backend { + lfs = fetchgit; + xet = throw "fetchFromHuggingFace: the Xet backend is not implemented yet"; + }; + in + assert ( + lib.assertMsg ( + builtins.match "[^/]+(/[^/]+)?" repoId != null + ) "fetchFromHuggingFace requires `repoId` to be in the form `repo` or `owner/repo`." + ); + backendFetcher ( + removeAttrs args [ + "backend" + "domain" + "repoId" + "repoType" + ] + // { + inherit + branchName + deepClone + fetchSubmodules + fetchTags + leaveDotGit + name + rootDir + sparseCheckout + tag + rev + ; + url = gitRepoUrl; + fetchLFS = true; + meta = newMeta; + passthru = { + inherit gitRepoUrl; + } + // passthru; + } + ) + // { + inherit + repoId + repoType + ; + } +) diff --git a/pkgs/build-support/fetchhuggingface/tests.nix b/pkgs/build-support/fetchhuggingface/tests.nix new file mode 100644 index 000000000000..3503af5fbdf9 --- /dev/null +++ b/pkgs/build-support/fetchhuggingface/tests.nix @@ -0,0 +1,163 @@ +{ + lib, + testers, + fetchFromHuggingFace, + runCommand, + ... +}: +let + fetchWithLFS = args: fetchFromHuggingFace (args // { backend = "lfs"; }); + fetchTestRepository = testers.invalidateFetcherByDrvHash fetchWithLFS; + fakeRev = "0123456789abcdef0123456789abcdef01234567"; + + expectEvalFailure = + name: expr: + let + result = builtins.tryEval expr; + in + runCommand "${name}-test" { } '' + test "${if result.success then "1" else "0"}" = "0" + touch "$out" + ''; +in +{ + apiSurface = + let + unnamespaced = fetchWithLFS { + repoId = "gpt2"; + rev = fakeRev; + hash = lib.fakeHash; + }; + tagged = fetchWithLFS { + repoId = "kitten/tagged-model"; + tag = "v1.0"; + hash = lib.fakeHash; + }; + dataset = fetchWithLFS { + repoId = "kitten/dataset"; + repoType = "dataset"; + domain = "hf.example"; + rev = fakeRev; + hash = lib.fakeHash; + }; + space = fetchWithLFS { + repoId = "kitten/space"; + repoType = "space"; + rev = fakeRev; + hash = lib.fakeHash; + passthru.custom = "value"; + }; + fetchgitOptions = fetchWithLFS { + repoId = "kitten/fetchgit-options"; + rev = fakeRev; + branchName = "huggingface"; + deepClone = true; + fetchSubmodules = true; + fetchTags = true; + leaveDotGit = true; + hash = lib.fakeHash; + }; + in + runCommand "fetchFromHuggingFace-api-surface-test" { } '' + test "${unnamespaced.repoId}" = "gpt2" + test "${unnamespaced.repoType}" = "model" + test "${unnamespaced.passthru.gitRepoUrl}" = "https://huggingface.co/gpt2.git" + test "${unnamespaced.meta.homepage}" = "https://huggingface.co/gpt2" + test "${if unnamespaced.fetchLFS then "1" else "0"}" = "1" + + test "${tagged.tag}" = "v1.0" + test "${tagged.rev}" = "refs/tags/v1.0" + test "${tagged.passthru.gitRepoUrl}" = "https://huggingface.co/kitten/tagged-model.git" + + test "${dataset.repoId}" = "kitten/dataset" + test "${dataset.repoType}" = "dataset" + test "${dataset.passthru.gitRepoUrl}" = "https://hf.example/datasets/kitten/dataset.git" + test "${dataset.meta.homepage}" = "https://hf.example/datasets/kitten/dataset" + + test "${space.repoType}" = "space" + test "${space.passthru.gitRepoUrl}" = "https://huggingface.co/spaces/kitten/space.git" + test "${if space.fetchLFS then "1" else "0"}" = "1" + test "${space.passthru.custom}" = "value" + + test "${fetchgitOptions.branchName}" = "huggingface" + test "${if fetchgitOptions.deepClone then "1" else "0"}" = "1" + test "${if fetchgitOptions.fetchLFS then "1" else "0"}" = "1" + test "${if fetchgitOptions.fetchSubmodules then "1" else "0"}" = "1" + test "${if fetchgitOptions.fetchTags then "1" else "0"}" = "1" + test "${if fetchgitOptions.leaveDotGit then "1" else "0"}" = "1" + + touch "$out" + ''; + + missingRevOrTag = expectEvalFailure "fetchFromHuggingFace-missing-rev-or-tag" ( + (fetchFromHuggingFace { + repoId = "gpt2"; + hash = lib.fakeHash; + }).drvPath + ); + + bothRevAndTag = expectEvalFailure "fetchFromHuggingFace-both-rev-and-tag" ( + (fetchFromHuggingFace { + repoId = "gpt2"; + rev = fakeRev; + tag = "main"; + hash = lib.fakeHash; + }).drvPath + ); + + invalidRepoId = expectEvalFailure "fetchFromHuggingFace-invalid-repo-id" ( + (fetchFromHuggingFace { + repoId = "broken/repo/id"; + rev = fakeRev; + hash = lib.fakeHash; + }).drvPath + ); + + invalidRepoType = expectEvalFailure "fetchFromHuggingFace-invalid-repo-type" ( + (fetchFromHuggingFace { + repoId = "gpt2"; + repoType = "collection"; + rev = fakeRev; + hash = lib.fakeHash; + }).drvPath + ); + + defaultXetBackend = expectEvalFailure "fetchFromHuggingFace-default-xet-backend" ( + (fetchFromHuggingFace { + repoId = "gpt2"; + rev = fakeRev; + hash = lib.fakeHash; + }).drvPath + ); + + explicitXetBackend = expectEvalFailure "fetchFromHuggingFace-explicit-xet-backend" ( + (fetchFromHuggingFace { + repoId = "gpt2"; + rev = fakeRev; + backend = "xet"; + hash = lib.fakeHash; + }).drvPath + ); + + invalidBackend = expectEvalFailure "fetchFromHuggingFace-invalid-backend" ( + (fetchFromHuggingFace { + repoId = "gpt2"; + rev = fakeRev; + backend = "git"; + hash = lib.fakeHash; + }).drvPath + ); + + simple = fetchTestRepository { + repoId = "hf-internal-testing/tiny-random-gpt2"; + rev = "71034c5d8bde858ff824298bdedc65515b97d2b9"; + hash = "sha256-8K9B/C62GW5lXC0c8QQpQ9QAE1UMoG+kYqvGhnWIp64="; + }; + + rootDir = fetchTestRepository { + repoId = "hf-internal-testing/tiny-random-BertModel"; + rev = "fc08ad9cc33be9aef4f55cc80e16ef5ae3d5981c"; + rootDir = "onnx"; + hash = "sha256-ETm2DT9jvVJ5W3MP8T0RiulNUlXlA2chtc9AVI+u6n4="; + }; +} diff --git a/pkgs/test/default.nix b/pkgs/test/default.nix index d02f45fb4884..35f3f2c8aa82 100644 --- a/pkgs/test/default.nix +++ b/pkgs/test/default.nix @@ -153,6 +153,9 @@ in ); fetchFromBitbucket = recurseIntoAttrs (callPackages ../build-support/fetchbitbucket/tests.nix { }); fetchFromGitHub = recurseIntoAttrs (callPackages ../build-support/fetchgithub/tests.nix { }); + fetchFromHuggingFace = recurseIntoAttrs ( + callPackages ../build-support/fetchhuggingface/tests.nix { } + ); fetchFirefoxAddon = recurseIntoAttrs ( callPackages ../build-support/fetchfirefoxaddon/tests.nix { } ); diff --git a/pkgs/top-level/all-packages.nix b/pkgs/top-level/all-packages.nix index 4f618b2918f0..6d1176e60ba4 100644 --- a/pkgs/top-level/all-packages.nix +++ b/pkgs/top-level/all-packages.nix @@ -642,6 +642,8 @@ with pkgs; fetchFromGitHub = callPackage ../build-support/fetchgithub { }; + fetchFromHuggingFace = callPackage ../build-support/fetchhuggingface { }; + fetchFromBitbucket = callPackage ../build-support/fetchbitbucket { }; fetchFromSavannah = callPackage ../build-support/fetchsavannah { };