From 970d249d39ccac356c705e6e4db6318ffe93dd42 Mon Sep 17 00:00:00 2001 From: Simon Bruder Date: Sun, 14 Nov 2021 10:07:09 +0100 Subject: [PATCH 1/7] invidious: unstable-2021-11-08 -> unstable-2021-11-13 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This update disables QUIC by default which fixes Invidious not loading anything except for the home page due to YouTube no longer accepting HTTP/3 (Upstream Issue: https://github.com/iv-org/invidious/issues/2577). It therefore uses Crystal’s internal HTTP client, which failed because the statically linked boringssl (required by lsquic) overrides OpenSSL’s CA certificate file location. This is fixed by applying the same patch to boringssl that is applied to openssl for using the correct CA certificate file. --- pkgs/servers/invidious/default.nix | 6 +++--- pkgs/servers/invidious/lsquic.nix | 5 +++++ pkgs/servers/invidious/use-etc-ssl-certs.patch | 13 +++++++++++++ 3 files changed, 21 insertions(+), 3 deletions(-) create mode 100644 pkgs/servers/invidious/use-etc-ssl-certs.patch diff --git a/pkgs/servers/invidious/default.nix b/pkgs/servers/invidious/default.nix index 7e939f51bc95..a50c128b03f5 100644 --- a/pkgs/servers/invidious/default.nix +++ b/pkgs/servers/invidious/default.nix @@ -8,17 +8,17 @@ let # * shards.nix (by running `crystal2nix` in invidious’ source tree) # * If the lsquic.cr dependency changed: lsquic in lsquic.nix (version, sha256) # * If the lsquic version changed: boringssl' in lsquic.nix (version, sha256) - rev = "21879da80d2dfa97e789a13b90e82e466c4854e3"; + rev = "00904ae3f2ab6a3cf5f96012d36c5672c3aa17b4"; in crystal.buildCrystalPackage rec { pname = "invidious"; - version = "unstable-2021-11-08"; + version = "unstable-2021-11-13"; src = fetchFromGitHub { owner = "iv-org"; repo = pname; inherit rev; - sha256 = "0jvnwjdh2l0hxfvzim00r3zbs528bb93y1nk0bjrbbrcfv5cn5ss"; + sha256 = "sha256-DET4jvB5epkpl5/HTORNTWDL4Ck4IsqhdTApJE8t6Tg="; }; postPatch = diff --git a/pkgs/servers/invidious/lsquic.nix b/pkgs/servers/invidious/lsquic.nix index 35b93e4249f2..fbef8805bb53 100644 --- a/pkgs/servers/invidious/lsquic.nix +++ b/pkgs/servers/invidious/lsquic.nix @@ -8,6 +8,11 @@ let rev = version; sha256 = "sha256-EU6T9yQCdOLx98Io8o01rEsgxDFF/Xoy42LgPopD2/A="; }; + + patches = [ + # Use /etc/ssl/certs/ca-certificates.crt instead of /etc/ssl/cert.pem + ./use-etc-ssl-certs.patch + ]; }); in stdenv.mkDerivation rec { diff --git a/pkgs/servers/invidious/use-etc-ssl-certs.patch b/pkgs/servers/invidious/use-etc-ssl-certs.patch new file mode 100644 index 000000000000..b60b0b1f1a12 --- /dev/null +++ b/pkgs/servers/invidious/use-etc-ssl-certs.patch @@ -0,0 +1,13 @@ +diff --git a/crypto/x509/x509_def.c b/crypto/x509/x509_def.c +index d2bc3e5c1..329580075 100644 +--- a/crypto/x509/x509_def.c ++++ b/crypto/x509/x509_def.c +@@ -67,7 +67,7 @@ + + #define X509_CERT_AREA OPENSSLDIR + #define X509_CERT_DIR OPENSSLDIR "/certs" +-#define X509_CERT_FILE OPENSSLDIR "/cert.pem" ++#define X509_CERT_FILE "/etc/ssl/certs/ca-certificates.crt" + #define X509_PRIVATE_DIR OPENSSLDIR "/private" + #define X509_CERT_DIR_EVP "SSL_CERT_DIR" + #define X509_CERT_FILE_EVP "SSL_CERT_FILE" From d697ee65ebec9c26250bc1e272ee61b8cffb650d Mon Sep 17 00:00:00 2001 From: schnusch Date: Wed, 15 Dec 2021 15:38:00 +0100 Subject: [PATCH 2/7] invidious: add update script based on mpvScripts.sponsorblock's update script --- pkgs/servers/invidious/default.nix | 6 ++- pkgs/servers/invidious/update.sh | 84 ++++++++++++++++++++++++++++++ 2 files changed, 89 insertions(+), 1 deletion(-) create mode 100755 pkgs/servers/invidious/update.sh diff --git a/pkgs/servers/invidious/default.nix b/pkgs/servers/invidious/default.nix index a50c128b03f5..ef4df055adb2 100644 --- a/pkgs/servers/invidious/default.nix +++ b/pkgs/servers/invidious/default.nix @@ -88,7 +88,11 @@ crystal.buildCrystalPackage rec { INVIDIOUS_CONFIG="database_url: sqlite3:///dev/null" $out/bin/invidious --help ''; - passthru.tests = { inherit (nixosTests) invidious; }; + passthru = { + inherit lsquic; + tests = { inherit (nixosTests) invidious; }; + updateScript = ./update.sh; + }; meta = with lib; { description = "An open source alternative front-end to YouTube"; diff --git a/pkgs/servers/invidious/update.sh b/pkgs/servers/invidious/update.sh new file mode 100755 index 000000000000..2824f8aaab98 --- /dev/null +++ b/pkgs/servers/invidious/update.sh @@ -0,0 +1,84 @@ +#!/usr/bin/env nix-shell +#!nix-shell -i bash -p common-updater-scripts curl crystal2nix jq git gnused nix nix-prefetch-git nix-update +git_url='https://github.com/iv-org/invidious.git' +git_branch='master' +git_dir='/var/tmp/invidious.git' +nix_file="$(dirname "${BASH_SOURCE[0]}")/default.nix" +pkg='invidious' + +set -euo pipefail + +info() { + if [ -t 2 ]; then + set -- '\033[32m%s\033[39m\n' "$@" + else + set -- '%s\n' "$@" + fi + printf "$@" >&2 +} + +old_rev=$(nix-instantiate --eval --strict --json -A "$pkg.src.rev" | jq -r) +old_version=$(nix-instantiate --eval --strict --json -A "$pkg.version" | jq -r) +today=$(LANG=C date -u +'%Y-%m-%d') + +info "fetching $git_url..." +if [ ! -d "$git_dir" ]; then + git init --initial-branch="$git_branch" "$git_dir" + git -C "$git_dir" remote add origin "$git_url" +fi +git -C "$git_dir" fetch origin "$git_branch" + +# use latest commit before today, we should not call the version *today* +# because there might still be commits coming +# use the day of the latest commit we picked as version +new_rev=$(git -C "$git_dir" log -n 1 --format='format:%H' --before="${today}T00:00:00Z" "origin/$git_branch") +new_version="unstable-$(git -C "$git_dir" log -n 1 --format='format:%cs' "$new_rev")" +info "latest commit before $today: $new_rev" + +if [ "$new_rev" = "$old_rev" ]; then + info "$pkg is up-to-date." + exit +fi + +new_sha256=$(nix-prefetch-git --rev "$new_rev" "$git_dir" | jq -r .sha256) +update-source-version "$pkg" \ + "$new_version" \ + "$new_sha256" \ + --rev="$new_rev" +git add "$nix_file" +commit_msg="$pkg: $old_version -> $new_version" + +cd "${nix_file%/*}" +if git -C "$git_dir" diff-tree --quiet "${old_rev}..${new_rev}" -- 'shard.lock'; then + info "shard.lock did not change since $old_rev." +else + info "Updating shards.nix..." + git -C "$git_dir" reset --hard "$new_rev" + crystal2nix -- "$git_dir/shard.lock" # argv's index seems broken + git add 'shards.nix' + + lsquic_old_version=$(nix-instantiate --eval --strict --json -A "${pkg}.lsquic.version" '../../..' | jq -r) + lsquic_new_version=$(nix eval --raw -f 'shards.nix' lsquic.rev \ + | sed -e 's/^v//' -e 's/-[0-9]*$//') + if [ "$lsquic_old_version" != "$lsquic_new_version" ]; then + info "Updating lsquic to $lsquic_new_version..." + nix-update --version "$lsquic_new_version" -f '../../..' invidious.lsquic + if git diff-index --quiet HEAD -- 'lsquic.nix'; then + info "lsquic is up-to-date." + else + boringssl_new_version=$(curl -LSsf "https://github.com/litespeedtech/lsquic/raw/v${lsquic_new_version}/README.md" \ + | grep -Pom1 '(?<=^git checkout ).*') + boringssl_new_sha256=$(nix-prefetch-git --rev "$boringssl_new_version" 'https://boringssl.googlesource.com/boringssl' \ + | jq -r .sha256) + sed -e "0,/^ *version = .*/ s// version = \"$boringssl_new_version\";/" \ + -e "0,/^ *sha256 = .*/ s// sha256 = \"$boringssl_new_sha256\";/" \ + -i lsquic.nix + git add 'lsquic.nix' + commit_msg="$commit_msg + +lsquic: $lsquic_old_version -> $lsquic_new_version" + fi + fi +fi + +git commit --verbose --message "$commit_msg" From eeacdb911a78dc6c8f83401a364b8d95eb2c802f Mon Sep 17 00:00:00 2001 From: schnusch Date: Sat, 15 Jan 2022 17:24:18 +0100 Subject: [PATCH 3/7] invidious: unstable-2021-11-13 -> unstable-2022-02-25 --- pkgs/servers/invidious/default.nix | 21 +++++++++++++-------- pkgs/servers/invidious/shards.nix | 12 ++++++++++++ pkgs/servers/invidious/update.sh | 22 +++++++++++++--------- pkgs/servers/invidious/videojs.nix | 15 +++++++++++++++ pkgs/servers/invidious/videojs.sh | 9 +++++++++ pkgs/top-level/all-packages.nix | 2 ++ 6 files changed, 64 insertions(+), 17 deletions(-) create mode 100644 pkgs/servers/invidious/videojs.nix create mode 100644 pkgs/servers/invidious/videojs.sh diff --git a/pkgs/servers/invidious/default.nix b/pkgs/servers/invidious/default.nix index ef4df055adb2..64ecba5cc882 100644 --- a/pkgs/servers/invidious/default.nix +++ b/pkgs/servers/invidious/default.nix @@ -1,4 +1,4 @@ -{ lib, crystal, fetchFromGitHub, librsvg, pkg-config, libxml2, openssl, sqlite, lsquic, nixosTests }: +{ lib, crystal, fetchFromGitHub, librsvg, pkg-config, libxml2, openssl, shards, sqlite, lsquic, videojs, nixosTests }: let # When updating, always update the following: # * the git revision @@ -8,17 +8,17 @@ let # * shards.nix (by running `crystal2nix` in invidious’ source tree) # * If the lsquic.cr dependency changed: lsquic in lsquic.nix (version, sha256) # * If the lsquic version changed: boringssl' in lsquic.nix (version, sha256) - rev = "00904ae3f2ab6a3cf5f96012d36c5672c3aa17b4"; + rev = "081fd541afc9b2f9b821e0f8f4c66dda0839295c"; in crystal.buildCrystalPackage rec { pname = "invidious"; - version = "unstable-2021-11-13"; + version = "unstable-2022-02-25"; src = fetchFromGitHub { owner = "iv-org"; repo = pname; inherit rev; - sha256 = "sha256-DET4jvB5epkpl5/HTORNTWDL4Ck4IsqhdTApJE8t6Tg="; + sha256 = "12m1fd8yfs6fqchvf9masr837dcghsg5x2nb8vcpzakzia5qc2kf"; }; postPatch = @@ -33,6 +33,8 @@ crystal.buildCrystalPackage rec { assetCommitTemplate = ''{{ "#{`git rev-list HEAD --max-count=1 --abbrev-commit -- assets`.strip}" }}''; in '' + for d in ${videojs}/*; do ln -s "$d" assets/videojs; done + # Use the version metadata from the derivation instead of using git at # build-time substituteInPlace src/invidious.cr \ @@ -48,19 +50,22 @@ crystal.buildCrystalPackage rec { --replace 'File.read("locales/' 'File.read("${placeholder "out"}/share/invidious/locales/' # Reference sql initialisation/migration scripts by absolute path - substituteInPlace src/invidious/helpers/helpers.cr \ + substituteInPlace src/invidious/database/base.cr \ --replace 'config/sql' '${placeholder "out"}/share/invidious/config/sql' - substituteInPlace src/invidious/users.cr \ + substituteInPlace src/invidious/user/captcha.cr \ --replace 'Process.run(%(rsvg-convert' 'Process.run(%(${lib.getBin librsvg}/bin/rsvg-convert' ''; - nativeBuildInputs = [ pkg-config ]; + nativeBuildInputs = [ pkg-config shards ]; buildInputs = [ libxml2 openssl sqlite ]; format = "crystal"; shardsFile = ./shards.nix; - crystalBinaries.invidious.src = "src/invidious.cr"; + crystalBinaries.invidious = { + src = "src/invidious.cr"; + options = [ "--release" "--progress" "--verbose" "--no-debug" "-Dskip_videojs_download" ]; + }; postConfigure = '' # lib includes nix store paths which can’t be patched, so the links have to diff --git a/pkgs/servers/invidious/shards.nix b/pkgs/servers/invidious/shards.nix index 778b34496a62..582a4083e7db 100644 --- a/pkgs/servers/invidious/shards.nix +++ b/pkgs/servers/invidious/shards.nix @@ -59,10 +59,22 @@ rev = "v0.4.1"; sha256 = "1l08cydkdidq9yyil1wl240hvk41iycv04jrg6nx5mkvzw4z1bzg"; }; + spectator = { + owner = "icy-arctic-fox"; + repo = "spectator"; + rev = "v0.10.4"; + sha256 = "0rcxq2nbslvwrd8m9ajw6dzaw3hagxmkdy9s8p34cgnr4c9dijdq"; + }; sqlite3 = { owner = "crystal-lang"; repo = "crystal-sqlite3"; rev = "v0.18.0"; sha256 = "03nnvpchhq9f9ywsm3pk2rrj4a3figw7xs96zdziwgr5znkz6x93"; }; + ameba = { + owner = "crystal-ameba"; + repo = "ameba"; + rev = "v0.14.3"; + sha256 = "1cfr95xi6hsyxw1wlrh571hc775xhwmssk3k14i8b7dgbwfmm5x1"; + }; } diff --git a/pkgs/servers/invidious/update.sh b/pkgs/servers/invidious/update.sh index 2824f8aaab98..bb85a7ba2ae0 100755 --- a/pkgs/servers/invidious/update.sh +++ b/pkgs/servers/invidious/update.sh @@ -1,9 +1,8 @@ #!/usr/bin/env nix-shell -#!nix-shell -i bash -p common-updater-scripts curl crystal2nix jq git gnused nix nix-prefetch-git nix-update +#!nix-shell -i bash -p common-updater-scripts curl crystal crystal2nix jq git gnused nix nix-prefetch-git nix-update pkg-config git_url='https://github.com/iv-org/invidious.git' git_branch='master' git_dir='/var/tmp/invidious.git' -nix_file="$(dirname "${BASH_SOURCE[0]}")/default.nix" pkg='invidious' set -euo pipefail @@ -45,17 +44,23 @@ update-source-version "$pkg" \ "$new_version" \ "$new_sha256" \ --rev="$new_rev" -git add "$nix_file" commit_msg="$pkg: $old_version -> $new_version" -cd "${nix_file%/*}" +cd "$(dirname "${BASH_SOURCE[0]}")" + +# fetch video.js dependencies +info "Running scripts/fetch-player-dependencies.cr..." +git -C "$git_dir" reset --hard "$new_rev" +(cd "$git_dir" && crystal run scripts/fetch-player-dependencies.cr -- --minified) +rm -f "$git_dir/assets/videojs/.gitignore" +videojs_new_sha256=$(nix hash-path --type sha256 --base32 "$git_dir/assets/videojs") +sed -e "s,\boutputHash = .*,outputHash = \"$videojs_new_sha256\";," -i 'videojs.nix' + if git -C "$git_dir" diff-tree --quiet "${old_rev}..${new_rev}" -- 'shard.lock'; then info "shard.lock did not change since $old_rev." else info "Updating shards.nix..." - git -C "$git_dir" reset --hard "$new_rev" crystal2nix -- "$git_dir/shard.lock" # argv's index seems broken - git add 'shards.nix' lsquic_old_version=$(nix-instantiate --eval --strict --json -A "${pkg}.lsquic.version" '../../..' | jq -r) lsquic_new_version=$(nix eval --raw -f 'shards.nix' lsquic.rev \ @@ -72,8 +77,7 @@ else | jq -r .sha256) sed -e "0,/^ *version = .*/ s// version = \"$boringssl_new_version\";/" \ -e "0,/^ *sha256 = .*/ s// sha256 = \"$boringssl_new_sha256\";/" \ - -i lsquic.nix - git add 'lsquic.nix' + -i 'lsquic.nix' commit_msg="$commit_msg lsquic: $lsquic_old_version -> $lsquic_new_version" @@ -81,4 +85,4 @@ lsquic: $lsquic_old_version -> $lsquic_new_version" fi fi -git commit --verbose --message "$commit_msg" +git commit --verbose --message "$commit_msg" *.nix diff --git a/pkgs/servers/invidious/videojs.nix b/pkgs/servers/invidious/videojs.nix new file mode 100644 index 000000000000..07358ad761bf --- /dev/null +++ b/pkgs/servers/invidious/videojs.nix @@ -0,0 +1,15 @@ +{ stdenvNoCC, cacert, crystal, openssl, pkg-config, invidious }: + +stdenvNoCC.mkDerivation { + name = "videojs"; + + inherit (invidious) src; + + builder = ./videojs.sh; + + nativeBuildInputs = [ cacert crystal openssl pkg-config ]; + + outputHashAlgo = "sha256"; + outputHashMode = "recursive"; + outputHash = "0b4vxd29kpvy60yhqm376r1872gds17s6wljqw0zlr16j762k50r"; +} diff --git a/pkgs/servers/invidious/videojs.sh b/pkgs/servers/invidious/videojs.sh new file mode 100644 index 000000000000..31609db14eb3 --- /dev/null +++ b/pkgs/servers/invidious/videojs.sh @@ -0,0 +1,9 @@ +source $stdenv/setup + +unpackPhase +cd source +# this helper downloads the videojs files and checks their checksums +# against videojs-dependencies.yml so it should be pure +crystal run scripts/fetch-player-dependencies.cr -- --minified +rm -f assets/videojs/.gitignore +mv assets/videojs "$out" diff --git a/pkgs/top-level/all-packages.nix b/pkgs/top-level/all-packages.nix index 35d58301dd8c..9f562c172c1a 100644 --- a/pkgs/top-level/all-packages.nix +++ b/pkgs/top-level/all-packages.nix @@ -6437,6 +6437,8 @@ with pkgs; invidious = callPackage ../servers/invidious { # needs a specific version of lsquic lsquic = callPackage ../servers/invidious/lsquic.nix { }; + # normally video.js is downloaded at build time + videojs = callPackage ../servers/invidious/videojs.nix { }; }; invoice2data = callPackage ../tools/text/invoice2data { }; From a4811f1fa3bb8324a7dd72e3bff0e3e29e4c4389 Mon Sep 17 00:00:00 2001 From: schnusch Date: Wed, 2 Mar 2022 11:53:08 +0100 Subject: [PATCH 4/7] invidious: move versions, revs and sha256s to JSON file This way updating these values is much less hacky. A downside is that tools like nix-update will no longer be able to update these, but this should not be too important for invidious. --- pkgs/servers/invidious/default.nix | 11 +++-- pkgs/servers/invidious/lsquic.nix | 15 ++++--- pkgs/servers/invidious/update.sh | 60 ++++++++++++++++------------ pkgs/servers/invidious/versions.json | 18 +++++++++ pkgs/servers/invidious/videojs.nix | 5 ++- 5 files changed, 70 insertions(+), 39 deletions(-) create mode 100644 pkgs/servers/invidious/versions.json diff --git a/pkgs/servers/invidious/default.nix b/pkgs/servers/invidious/default.nix index 64ecba5cc882..14eed9a3be71 100644 --- a/pkgs/servers/invidious/default.nix +++ b/pkgs/servers/invidious/default.nix @@ -8,17 +8,16 @@ let # * shards.nix (by running `crystal2nix` in invidious’ source tree) # * If the lsquic.cr dependency changed: lsquic in lsquic.nix (version, sha256) # * If the lsquic version changed: boringssl' in lsquic.nix (version, sha256) - rev = "081fd541afc9b2f9b821e0f8f4c66dda0839295c"; + versions = builtins.fromJSON (builtins.readFile ./versions.json); in crystal.buildCrystalPackage rec { pname = "invidious"; - version = "unstable-2022-02-25"; + inherit (versions.invidious) version; src = fetchFromGitHub { owner = "iv-org"; repo = pname; - inherit rev; - sha256 = "12m1fd8yfs6fqchvf9masr837dcghsg5x2nb8vcpzakzia5qc2kf"; + inherit (versions.invidious) rev sha256; }; postPatch = @@ -39,9 +38,9 @@ crystal.buildCrystalPackage rec { # build-time substituteInPlace src/invidious.cr \ --replace ${lib.escapeShellArg branchTemplate} '"master"' \ - --replace ${lib.escapeShellArg commitTemplate} '"${lib.substring 0 7 rev}"' \ + --replace ${lib.escapeShellArg commitTemplate} '"${lib.substring 0 7 versions.invidious.rev}"' \ --replace ${lib.escapeShellArg versionTemplate} '"${lib.replaceChars ["-"] ["."] (lib.substring 9 10 version)}"' \ - --replace ${lib.escapeShellArg assetCommitTemplate} '"${lib.substring 0 7 rev}"' + --replace ${lib.escapeShellArg assetCommitTemplate} '"${lib.substring 0 7 versions.invidious.rev}"' # Patch the assets and locales paths to be absolute substituteInPlace src/invidious.cr \ diff --git a/pkgs/servers/invidious/lsquic.nix b/pkgs/servers/invidious/lsquic.nix index fbef8805bb53..08df5b4a1d1f 100644 --- a/pkgs/servers/invidious/lsquic.nix +++ b/pkgs/servers/invidious/lsquic.nix @@ -1,12 +1,13 @@ { lib, boringssl, stdenv, fetchgit, fetchFromGitHub, cmake, zlib, perl, libevent }: let + versions = builtins.fromJSON (builtins.readFile ./versions.json); + # lsquic requires a specific boringssl version (noted in its README) - boringssl' = boringssl.overrideAttrs (old: rec { - version = "251b5169fd44345f455438312ec4e18ae07fd58c"; + boringssl' = boringssl.overrideAttrs (old: { + version = versions.boringssl.rev; src = fetchgit { url = "https://boringssl.googlesource.com/boringssl"; - rev = version; - sha256 = "sha256-EU6T9yQCdOLx98Io8o01rEsgxDFF/Xoy42LgPopD2/A="; + inherit (versions.boringssl) rev sha256; }; patches = [ @@ -17,13 +18,13 @@ let in stdenv.mkDerivation rec { pname = "lsquic"; - version = "2.18.1"; + version = versions.lsquic.version; src = fetchFromGitHub { owner = "litespeedtech"; repo = pname; rev = "v${version}"; - sha256 = "sha256-hG8cUvhbCNeMOsKkaJlgGpzUrIx47E/WhmPIdI5F3qM="; + inherit (versions.lsquic) sha256; fetchSubmodules = true; }; @@ -54,6 +55,8 @@ stdenv.mkDerivation rec { runHook postInstall ''; + passthru.boringssl = boringssl'; + meta = with lib; { description = "A library for QUIC and HTTP/3 (version for Invidious)"; homepage = "https://github.com/litespeedtech/lsquic"; diff --git a/pkgs/servers/invidious/update.sh b/pkgs/servers/invidious/update.sh index bb85a7ba2ae0..580d6136388a 100755 --- a/pkgs/servers/invidious/update.sh +++ b/pkgs/servers/invidious/update.sh @@ -1,5 +1,5 @@ #!/usr/bin/env nix-shell -#!nix-shell -i bash -p common-updater-scripts curl crystal crystal2nix jq git gnused nix nix-prefetch-git nix-update pkg-config +#!nix-shell -i bash -p curl crystal crystal2nix jq git moreutils nix nix-prefetch pkg-config git_url='https://github.com/iv-org/invidious.git' git_branch='master' git_dir='/var/tmp/invidious.git' @@ -7,6 +7,8 @@ pkg='invidious' set -euo pipefail +cd "$(dirname "${BASH_SOURCE[0]}")" + info() { if [ -t 2 ]; then set -- '\033[32m%s\033[39m\n' "$@" @@ -16,8 +18,16 @@ info() { printf "$@" >&2 } -old_rev=$(nix-instantiate --eval --strict --json -A "$pkg.src.rev" | jq -r) -old_version=$(nix-instantiate --eval --strict --json -A "$pkg.version" | jq -r) +json_get() { + jq -r "$1" < 'versions.json' +} + +json_set() { + jq --arg x "$2" "$1 = \$x" < 'versions.json' | sponge 'versions.json' +} + +old_rev=$(json_get '.invidious.rev') +old_version=$(json_get '.invidious.version') today=$(LANG=C date -u +'%Y-%m-%d') info "fetching $git_url..." @@ -39,22 +49,19 @@ if [ "$new_rev" = "$old_rev" ]; then exit fi -new_sha256=$(nix-prefetch-git --rev "$new_rev" "$git_dir" | jq -r .sha256) -update-source-version "$pkg" \ - "$new_version" \ - "$new_sha256" \ - --rev="$new_rev" +json_set '.invidious.version' "$new_version" +json_set '.invidious.rev' "$new_rev" +new_sha256=$(nix-prefetch -I 'nixpkgs=../../..' "$pkg") +json_set '.invidious.sha256' "$new_sha256" commit_msg="$pkg: $old_version -> $new_version" -cd "$(dirname "${BASH_SOURCE[0]}")" - # fetch video.js dependencies info "Running scripts/fetch-player-dependencies.cr..." git -C "$git_dir" reset --hard "$new_rev" (cd "$git_dir" && crystal run scripts/fetch-player-dependencies.cr -- --minified) rm -f "$git_dir/assets/videojs/.gitignore" videojs_new_sha256=$(nix hash-path --type sha256 --base32 "$git_dir/assets/videojs") -sed -e "s,\boutputHash = .*,outputHash = \"$videojs_new_sha256\";," -i 'videojs.nix' +json_set '.videojs.sha256' "$videojs_new_sha256" if git -C "$git_dir" diff-tree --quiet "${old_rev}..${new_rev}" -- 'shard.lock'; then info "shard.lock did not change since $old_rev." @@ -62,27 +69,28 @@ else info "Updating shards.nix..." crystal2nix -- "$git_dir/shard.lock" # argv's index seems broken - lsquic_old_version=$(nix-instantiate --eval --strict --json -A "${pkg}.lsquic.version" '../../..' | jq -r) + lsquic_old_version=$(json_get '.lsquic.version') + # lsquic.cr's version tracks lsquic's, so lsquic must be updated to the + # version in the shards file lsquic_new_version=$(nix eval --raw -f 'shards.nix' lsquic.rev \ | sed -e 's/^v//' -e 's/-[0-9]*$//') if [ "$lsquic_old_version" != "$lsquic_new_version" ]; then info "Updating lsquic to $lsquic_new_version..." - nix-update --version "$lsquic_new_version" -f '../../..' invidious.lsquic - if git diff-index --quiet HEAD -- 'lsquic.nix'; then - info "lsquic is up-to-date." - else - boringssl_new_version=$(curl -LSsf "https://github.com/litespeedtech/lsquic/raw/v${lsquic_new_version}/README.md" \ - | grep -Pom1 '(?<=^git checkout ).*') - boringssl_new_sha256=$(nix-prefetch-git --rev "$boringssl_new_version" 'https://boringssl.googlesource.com/boringssl' \ - | jq -r .sha256) - sed -e "0,/^ *version = .*/ s// version = \"$boringssl_new_version\";/" \ - -e "0,/^ *sha256 = .*/ s// sha256 = \"$boringssl_new_sha256\";/" \ - -i 'lsquic.nix' - commit_msg="$commit_msg + json_set '.lsquic.version' "$lsquic_new_version" + lsquic_new_sha256=$(nix-prefetch -I 'nixpkgs=../../..' "${pkg}.lsquic") + json_set '.lsquic.sha256' "$lsquic_new_sha256" + + info "Updating boringssl..." + # lsquic specifies the boringssl commit it requires in its README + boringssl_new_rev=$(curl -LSsf "https://github.com/litespeedtech/lsquic/raw/v${lsquic_new_version}/README.md" \ + | grep -Pom1 '(?<=^git checkout ).*') + json_set '.boringssl.rev' "$boringssl_new_rev" + boringssl_new_sha256=$(nix-prefetch -I 'nixpkgs=../../..' "${pkg}.lsquic.boringssl") + json_set '.boringssl.sha256' "$boringssl_new_sha256" + commit_msg="$commit_msg lsquic: $lsquic_old_version -> $lsquic_new_version" - fi fi fi -git commit --verbose --message "$commit_msg" *.nix +git commit --verbose --message "$commit_msg" -- versions.json shards.nix diff --git a/pkgs/servers/invidious/versions.json b/pkgs/servers/invidious/versions.json new file mode 100644 index 000000000000..46d3c1740783 --- /dev/null +++ b/pkgs/servers/invidious/versions.json @@ -0,0 +1,18 @@ +{ + "boringssl": { + "rev": "251b5169fd44345f455438312ec4e18ae07fd58c", + "sha256": "sha256-EU6T9yQCdOLx98Io8o01rEsgxDFF/Xoy42LgPopD2/A=" + }, + "invidious": { + "rev": "081fd541afc9b2f9b821e0f8f4c66dda0839295c", + "sha256": "12m1fd8yfs6fqchvf9masr837dcghsg5x2nb8vcpzakzia5qc2kf", + "version": "unstable-2022-02-25" + }, + "lsquic": { + "sha256": "sha256-hG8cUvhbCNeMOsKkaJlgGpzUrIx47E/WhmPIdI5F3qM=", + "version": "2.18.1" + }, + "videojs": { + "sha256": "0b4vxd29kpvy60yhqm376r1872gds17s6wljqw0zlr16j762k50r" + } +} diff --git a/pkgs/servers/invidious/videojs.nix b/pkgs/servers/invidious/videojs.nix index 07358ad761bf..e4470793e51e 100644 --- a/pkgs/servers/invidious/videojs.nix +++ b/pkgs/servers/invidious/videojs.nix @@ -1,5 +1,8 @@ { stdenvNoCC, cacert, crystal, openssl, pkg-config, invidious }: +let + versions = builtins.fromJSON (builtins.readFile ./versions.json); +in stdenvNoCC.mkDerivation { name = "videojs"; @@ -11,5 +14,5 @@ stdenvNoCC.mkDerivation { outputHashAlgo = "sha256"; outputHashMode = "recursive"; - outputHash = "0b4vxd29kpvy60yhqm376r1872gds17s6wljqw0zlr16j762k50r"; + outputHash = versions.videojs.sha256; } From 30bbba0e1ffc0c4803ab8ff412858d3d2b74b002 Mon Sep 17 00:00:00 2001 From: schnusch Date: Sun, 13 Mar 2022 11:35:06 +0100 Subject: [PATCH 5/7] invidious: document update process --- pkgs/servers/invidious/default.nix | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/pkgs/servers/invidious/default.nix b/pkgs/servers/invidious/default.nix index 14eed9a3be71..6e09678450a4 100644 --- a/pkgs/servers/invidious/default.nix +++ b/pkgs/servers/invidious/default.nix @@ -1,13 +1,17 @@ { lib, crystal, fetchFromGitHub, librsvg, pkg-config, libxml2, openssl, shards, sqlite, lsquic, videojs, nixosTests }: let - # When updating, always update the following: - # * the git revision - # * the version attribute - # * the source hash (sha256) - # If the shards.lock file changed, also the following: - # * shards.nix (by running `crystal2nix` in invidious’ source tree) - # * If the lsquic.cr dependency changed: lsquic in lsquic.nix (version, sha256) - # * If the lsquic version changed: boringssl' in lsquic.nix (version, sha256) + # All versions, revisions, and checksums are stored in ./versions.json. + # The update process is the following: + # * pick the latest commit + # * update .invidious.rev, .invidious.version, and .invidious.sha256 + # * prefetch the videojs dependencies with scripts/fetch-player-dependencies.cr + # and update .videojs.sha256 (they are normally fetched during build + # but nix's sandboxing does not allow that) + # * if shard.lock changed + # * recreate shards.nix by running crystal2nix + # * update lsquic and boringssl if necessarry, lsquic.cr depends on + # the same version of lsquic and lsquic requires the boringssl + # commit mentioned in its README versions = builtins.fromJSON (builtins.readFile ./versions.json); in crystal.buildCrystalPackage rec { From 268c01cef1d57b445cf587ae17fb111778e402e1 Mon Sep 17 00:00:00 2001 From: schnusch Date: Thu, 17 Mar 2022 00:26:08 +0100 Subject: [PATCH 6/7] invidious: mark as broken on aarch64-darwin Build fails with Failed to spawn codesign_allocate: No such file or directory --- pkgs/servers/invidious/default.nix | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pkgs/servers/invidious/default.nix b/pkgs/servers/invidious/default.nix index 6e09678450a4..adcd694bad94 100644 --- a/pkgs/servers/invidious/default.nix +++ b/pkgs/servers/invidious/default.nix @@ -1,4 +1,4 @@ -{ lib, crystal, fetchFromGitHub, librsvg, pkg-config, libxml2, openssl, shards, sqlite, lsquic, videojs, nixosTests }: +{ lib, stdenv, crystal, fetchFromGitHub, librsvg, pkg-config, libxml2, openssl, shards, sqlite, lsquic, videojs, nixosTests }: let # All versions, revisions, and checksums are stored in ./versions.json. # The update process is the following: @@ -107,5 +107,6 @@ crystal.buildCrystalPackage rec { homepage = "https://invidious.io/"; license = licenses.agpl3; maintainers = with maintainers; [ infinisil sbruder ]; + broken = stdenv.isDarwin && stdenv.isAarch64; }; } From c418d051908a692ec69960fc09c2cb70e6c570e3 Mon Sep 17 00:00:00 2001 From: schnusch Date: Fri, 4 Mar 2022 07:39:51 +0100 Subject: [PATCH 7/7] invidious: unstable-2022-02-25 -> unstable-2022-03-16 --- pkgs/servers/invidious/versions.json | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pkgs/servers/invidious/versions.json b/pkgs/servers/invidious/versions.json index 46d3c1740783..cec068a09bf1 100644 --- a/pkgs/servers/invidious/versions.json +++ b/pkgs/servers/invidious/versions.json @@ -4,9 +4,9 @@ "sha256": "sha256-EU6T9yQCdOLx98Io8o01rEsgxDFF/Xoy42LgPopD2/A=" }, "invidious": { - "rev": "081fd541afc9b2f9b821e0f8f4c66dda0839295c", - "sha256": "12m1fd8yfs6fqchvf9masr837dcghsg5x2nb8vcpzakzia5qc2kf", - "version": "unstable-2022-02-25" + "rev": "ed265cfdcd131b9df5398d899cc5d7036a5b7846", + "sha256": "0hhnq4s0slwbgxra7gxapl7dcz60a7k71cndi4crqcikmazzac3b", + "version": "unstable-2022-03-16" }, "lsquic": { "sha256": "sha256-hG8cUvhbCNeMOsKkaJlgGpzUrIx47E/WhmPIdI5F3qM=",