From 71cda4fbef0c064b4df82ac65dd2cc868bb37c32 Mon Sep 17 00:00:00 2001 From: Junji Hashimoto Date: Fri, 27 Aug 2021 16:52:40 +0900 Subject: [PATCH 1/5] Bump the binaries of torchvision --- maintainers/maintainer-list.nix | 6 +++ .../python-modules/torchvision/bin.nix | 4 +- .../torchvision/binary-hashes.nix | 49 +++++++++++++------ .../python-modules/torchvision/prefetch.sh | 35 +++++++++++++ 4 files changed, 78 insertions(+), 16 deletions(-) create mode 100755 pkgs/development/python-modules/torchvision/prefetch.sh diff --git a/maintainers/maintainer-list.nix b/maintainers/maintainer-list.nix index 43968e4d80d8..5401fbe800f2 100644 --- a/maintainers/maintainer-list.nix +++ b/maintainers/maintainer-list.nix @@ -5408,6 +5408,12 @@ githubId = 39434424; name = "Felix Springer"; }; + junjihashimoto = { + email = "junji.hashimoto@gmail.com"; + github = "junjihashimoto"; + githubId = 2469618; + name = "Junji Hashimoto"; + }; justinas = { email = "justinas@justinas.org"; github = "justinas"; diff --git a/pkgs/development/python-modules/torchvision/bin.nix b/pkgs/development/python-modules/torchvision/bin.nix index 62980dcca8c4..35d2f8f13252 100644 --- a/pkgs/development/python-modules/torchvision/bin.nix +++ b/pkgs/development/python-modules/torchvision/bin.nix @@ -9,13 +9,13 @@ , pillow , python , pytorch-bin +, version ? "0.10.0" }: let pyVerNoDot = builtins.replaceStrings [ "." ] [ "" ] python.pythonVersion; srcs = import ./binary-hashes.nix version; unsupported = throw "Unsupported system"; - version = "0.9.1"; in buildPythonPackage { inherit version; @@ -54,6 +54,6 @@ in buildPythonPackage { changelog = "https://github.com/pytorch/vision/releases/tag/v${version}"; license = licenses.bsd3; platforms = platforms.linux; - maintainers = with maintainers; [ danieldk ]; + maintainers = with maintainers; [ danieldk junjihashimoto ]; }; } diff --git a/pkgs/development/python-modules/torchvision/binary-hashes.nix b/pkgs/development/python-modules/torchvision/binary-hashes.nix index b9a991e24abb..0afd0a2f3a3a 100644 --- a/pkgs/development/python-modules/torchvision/binary-hashes.nix +++ b/pkgs/development/python-modules/torchvision/binary-hashes.nix @@ -3,20 +3,41 @@ # Precompiled wheels can be found at: # https://download.pytorch.org/whl/torch_stable.html -version: { - x86_64-linux-37 = { - name = "torchvision-${version}-cp37-cp37m-linux_x86_64.whl"; - url = "https://download.pytorch.org/whl/cu111/torchvision-${version}%2Bcu111-cp37-cp37m-linux_x86_64.whl"; - hash = "sha256-7EMVB8KZg2I3P4RqnIVk/7OOAPA1OWOipns58cSCUrw="; +# To add a new version, run "prefetch.sh 'new-version'" to paste the generated file as follows. + +version : builtins.getAttr version { + "0.9.1" = { + x86_64-linux-37 = { + name = "torchvision-0.9.1-cp37-cp37m-linux_x86_64.whl"; + url = "https://download.pytorch.org/whl/cu111/torchvision-0.9.1%2Bcu111-cp37-cp37m-linux_x86_64.whl"; + hash = "sha256-7EMVB8KZg2I3P4RqnIVk/7OOAPA1OWOipns58cSCUrw="; + }; + x86_64-linux-38 = { + name = "torchvision-0.9.1-cp38-cp38-linux_x86_64.whl"; + url = "https://download.pytorch.org/whl/cu111/torchvision-0.9.1%2Bcu111-cp38-cp38-linux_x86_64.whl"; + hash = "sha256-VjsCBW9Lusr4aDQLqaFh5dpV/5ZJ5PDs7nY4CbCHDTA="; + }; + x86_64-linux-39 = { + name = "torchvision-0.9.1-cp39-cp39-linux_x86_64.whl"; + url = "https://download.pytorch.org/whl/cu111/torchvision-0.9.1%2Bcu111-cp39-cp39-linux_x86_64.whl"; + hash = "sha256-pzR7TBE+WcAmozskoeOVBuMkGJf9tvsaXsUkTcu86N8="; + }; }; - x86_64-linux-38 = { - name = "torchvision-${version}-cp38-cp38-linux_x86_64.whl"; - url = "https://download.pytorch.org/whl/cu111/torchvision-${version}%2Bcu111-cp38-cp38-linux_x86_64.whl"; - hash = "sha256-VjsCBW9Lusr4aDQLqaFh5dpV/5ZJ5PDs7nY4CbCHDTA="; - }; - x86_64-linux-39 = { - name = "torchvision-${version}-cp39-cp39-linux_x86_64.whl"; - url = "https://download.pytorch.org/whl/cu111/torchvision-${version}%2Bcu111-cp39-cp39-linux_x86_64.whl"; - hash = "sha256-pzR7TBE+WcAmozskoeOVBuMkGJf9tvsaXsUkTcu86N8="; + "0.10.0" = { + x86_64-linux-37 = { + name = "torchvision-0.10.0-cp37-cp37m-linux_x86_64.whl"; + url = "https://download.pytorch.org/whl/cu111/torchvision-0.10.0%2Bcu111-cp37-cp37m-linux_x86_64.whl"; + sha256 = "1kh6zrid4jv2qh1fpgnghsy2p0k5dc9pmk90484bdziskn313j68"; + }; + x86_64-linux-38 = { + name = "torchvision-0.10.0-cp38-cp38-linux_x86_64.whl"; + url = "https://download.pytorch.org/whl/cu111/torchvision-0.10.0%2Bcu111-cp38-cp38-linux_x86_64.whl"; + sha256 = "1xxlkdjj85sryvah381byfa1xgsl21vc307gjnbknxdchbxz0mm7"; + }; + x86_64-linux-39 = { + name = "torchvision-0.10.0-cp39-cp39-linux_x86_64.whl"; + url = "https://download.pytorch.org/whl/cu111/torchvision-0.10.0%2Bcu111-cp39-cp39-linux_x86_64.whl"; + sha256 = "18gcf9nj5cfvvm7qqhkiq4y5sgzrwfsdhqxcwrqx4mg1fjff3rq0"; + }; }; } diff --git a/pkgs/development/python-modules/torchvision/prefetch.sh b/pkgs/development/python-modules/torchvision/prefetch.sh new file mode 100755 index 000000000000..5cb754a7bd6e --- /dev/null +++ b/pkgs/development/python-modules/torchvision/prefetch.sh @@ -0,0 +1,35 @@ +#!/usr/bin/env bash + +version=$1 + +bucket="https://download.pytorch.org/whl/cu111" + +url_and_key_list=( +"x86_64-linux-37 $bucket/torchvision-${version}%2Bcu111-cp37-cp37m-linux_x86_64.whl torchvision-${version}-cp37-cp37m-linux_x86_64.whl" +"x86_64-linux-38 $bucket/torchvision-${version}%2Bcu111-cp38-cp38-linux_x86_64.whl torchvision-${version}-cp38-cp38-linux_x86_64.whl" +"x86_64-linux-39 $bucket/torchvision-${version}%2Bcu111-cp39-cp39-linux_x86_64.whl torchvision-${version}-cp39-cp39-linux_x86_64.whl" +) + +hashfile=binary-hashes-"$version".nix +rm -f $hashfile +echo " \"$version\" = {" >> $hashfile + +for url_and_key in "${url_and_key_list[@]}"; do + key=$(echo "$url_and_key" | cut -d' ' -f1) + url=$(echo "$url_and_key" | cut -d' ' -f2) + name=$(echo "$url_and_key" | cut -d' ' -f3) + + echo "prefetching ${url}..." + hash=$(nix-prefetch-url "$url" --name "$name") + + echo " $key = {" >> $hashfile + echo " name = \"$name\";" >> $hashfile + echo " url = \"$url\";" >> $hashfile + echo " sha256 = \"$hash\";" >> $hashfile + echo " };" >> $hashfile + + echo +done + +echo " };" >> $hashfile +echo "done." From 129020eda640b37a977ff260d91ad12dc4e0581d Mon Sep 17 00:00:00 2001 From: Junji Hashimoto Date: Tue, 14 Sep 2021 04:25:30 +0000 Subject: [PATCH 2/5] Change hash-type from base32 to base64 --- .../python-modules/torchvision/binary-hashes.nix | 6 +++--- pkgs/development/python-modules/torchvision/prefetch.sh | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/pkgs/development/python-modules/torchvision/binary-hashes.nix b/pkgs/development/python-modules/torchvision/binary-hashes.nix index 0afd0a2f3a3a..3b8fd7aaefa5 100644 --- a/pkgs/development/python-modules/torchvision/binary-hashes.nix +++ b/pkgs/development/python-modules/torchvision/binary-hashes.nix @@ -27,17 +27,17 @@ version : builtins.getAttr version { x86_64-linux-37 = { name = "torchvision-0.10.0-cp37-cp37m-linux_x86_64.whl"; url = "https://download.pytorch.org/whl/cu111/torchvision-0.10.0%2Bcu111-cp37-cp37m-linux_x86_64.whl"; - sha256 = "1kh6zrid4jv2qh1fpgnghsy2p0k5dc9pmk90484bdziskn313j68"; + hash = "sha256-yMgRhp06/rYIIiDNehNrZYIrvIbPvusCxGJL0mL+Bs4="; }; x86_64-linux-38 = { name = "torchvision-0.10.0-cp38-cp38-linux_x86_64.whl"; url = "https://download.pytorch.org/whl/cu111/torchvision-0.10.0%2Bcu111-cp38-cp38-linux_x86_64.whl"; - sha256 = "1xxlkdjj85sryvah381byfa1xgsl21vc307gjnbknxdchbxz0mm7"; + hash = "sha256-p1bw+4KsdTuXle+AwXYQVL8elPMroAHV9lkXJGWbtPc="; }; x86_64-linux-39 = { name = "torchvision-0.10.0-cp39-cp39-linux_x86_64.whl"; url = "https://download.pytorch.org/whl/cu111/torchvision-0.10.0%2Bcu111-cp39-cp39-linux_x86_64.whl"; - sha256 = "18gcf9nj5cfvvm7qqhkiq4y5sgzrwfsdhqxcwrqx4mg1fjff3rq0"; + hash = "sha256-AOfhnHThVdJx5qxj2LTj+T9dPMFxQoxP3duxIm1y7KE="; }; }; } diff --git a/pkgs/development/python-modules/torchvision/prefetch.sh b/pkgs/development/python-modules/torchvision/prefetch.sh index 5cb754a7bd6e..57d35e591154 100755 --- a/pkgs/development/python-modules/torchvision/prefetch.sh +++ b/pkgs/development/python-modules/torchvision/prefetch.sh @@ -20,12 +20,12 @@ for url_and_key in "${url_and_key_list[@]}"; do name=$(echo "$url_and_key" | cut -d' ' -f3) echo "prefetching ${url}..." - hash=$(nix-prefetch-url "$url" --name "$name") + hash=$(nix hash to-base64 --type sha256 `nix-prefetch-url "$url" --name "$name"`) echo " $key = {" >> $hashfile echo " name = \"$name\";" >> $hashfile echo " url = \"$url\";" >> $hashfile - echo " sha256 = \"$hash\";" >> $hashfile + echo " hash = \"sha256-$hash\";" >> $hashfile echo " };" >> $hashfile echo From 34e940938de9fdb0c8993841fe7a90f868b12bae Mon Sep 17 00:00:00 2001 From: Junji Hashimoto Date: Tue, 14 Sep 2021 04:33:32 +0000 Subject: [PATCH 3/5] Change nix-hash to nix-to-sri --- pkgs/development/python-modules/torchvision/prefetch.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkgs/development/python-modules/torchvision/prefetch.sh b/pkgs/development/python-modules/torchvision/prefetch.sh index 57d35e591154..caa987d84409 100755 --- a/pkgs/development/python-modules/torchvision/prefetch.sh +++ b/pkgs/development/python-modules/torchvision/prefetch.sh @@ -20,12 +20,12 @@ for url_and_key in "${url_and_key_list[@]}"; do name=$(echo "$url_and_key" | cut -d' ' -f3) echo "prefetching ${url}..." - hash=$(nix hash to-base64 --type sha256 `nix-prefetch-url "$url" --name "$name"`) + hash=$(nix hash to-sri --type sha256 `nix-prefetch-url "$url" --name "$name"`) echo " $key = {" >> $hashfile echo " name = \"$name\";" >> $hashfile echo " url = \"$url\";" >> $hashfile - echo " hash = \"sha256-$hash\";" >> $hashfile + echo " hash = \"$hash\";" >> $hashfile echo " };" >> $hashfile echo From 60f4fa1efbec6c0364b0de1157ec69efe03ff143 Mon Sep 17 00:00:00 2001 From: Junji Hashimoto Date: Tue, 14 Sep 2021 15:52:13 +0000 Subject: [PATCH 4/5] Use nix-shell for prefetch.sh of torchvision --- pkgs/development/python-modules/torchvision/prefetch.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pkgs/development/python-modules/torchvision/prefetch.sh b/pkgs/development/python-modules/torchvision/prefetch.sh index caa987d84409..f57ac96bdb63 100755 --- a/pkgs/development/python-modules/torchvision/prefetch.sh +++ b/pkgs/development/python-modules/torchvision/prefetch.sh @@ -1,4 +1,5 @@ -#!/usr/bin/env bash +#!/usr/bin/env nix-shell +#!nix-shell -i bash -p nix-prefetch-scripts version=$1 From cb5023dbf14e997935c83434b976f553aef642f0 Mon Sep 17 00:00:00 2001 From: Junji Hashimoto Date: Wed, 15 Sep 2021 09:00:43 +0000 Subject: [PATCH 5/5] Revert the argument of version, and remove 0.9.1 --- .../python-modules/torchvision/bin.nix | 2 +- .../torchvision/binary-hashes.nix | 17 ----------------- 2 files changed, 1 insertion(+), 18 deletions(-) diff --git a/pkgs/development/python-modules/torchvision/bin.nix b/pkgs/development/python-modules/torchvision/bin.nix index cac01cbcb596..e874d22c1029 100644 --- a/pkgs/development/python-modules/torchvision/bin.nix +++ b/pkgs/development/python-modules/torchvision/bin.nix @@ -9,13 +9,13 @@ , pillow , python , pytorch-bin -, version ? "0.10.0" }: let pyVerNoDot = builtins.replaceStrings [ "." ] [ "" ] python.pythonVersion; srcs = import ./binary-hashes.nix version; unsupported = throw "Unsupported system"; + version = "0.10.0"; in buildPythonPackage { inherit version; diff --git a/pkgs/development/python-modules/torchvision/binary-hashes.nix b/pkgs/development/python-modules/torchvision/binary-hashes.nix index 3b8fd7aaefa5..c9b5a4df42ea 100644 --- a/pkgs/development/python-modules/torchvision/binary-hashes.nix +++ b/pkgs/development/python-modules/torchvision/binary-hashes.nix @@ -6,23 +6,6 @@ # To add a new version, run "prefetch.sh 'new-version'" to paste the generated file as follows. version : builtins.getAttr version { - "0.9.1" = { - x86_64-linux-37 = { - name = "torchvision-0.9.1-cp37-cp37m-linux_x86_64.whl"; - url = "https://download.pytorch.org/whl/cu111/torchvision-0.9.1%2Bcu111-cp37-cp37m-linux_x86_64.whl"; - hash = "sha256-7EMVB8KZg2I3P4RqnIVk/7OOAPA1OWOipns58cSCUrw="; - }; - x86_64-linux-38 = { - name = "torchvision-0.9.1-cp38-cp38-linux_x86_64.whl"; - url = "https://download.pytorch.org/whl/cu111/torchvision-0.9.1%2Bcu111-cp38-cp38-linux_x86_64.whl"; - hash = "sha256-VjsCBW9Lusr4aDQLqaFh5dpV/5ZJ5PDs7nY4CbCHDTA="; - }; - x86_64-linux-39 = { - name = "torchvision-0.9.1-cp39-cp39-linux_x86_64.whl"; - url = "https://download.pytorch.org/whl/cu111/torchvision-0.9.1%2Bcu111-cp39-cp39-linux_x86_64.whl"; - hash = "sha256-pzR7TBE+WcAmozskoeOVBuMkGJf9tvsaXsUkTcu86N8="; - }; - }; "0.10.0" = { x86_64-linux-37 = { name = "torchvision-0.10.0-cp37-cp37m-linux_x86_64.whl";