diff --git a/doc/languages-frameworks/crystal.section.md b/doc/languages-frameworks/crystal.section.md index cbe31f9f0b2f..cbabba24f0c1 100644 --- a/doc/languages-frameworks/crystal.section.md +++ b/doc/languages-frameworks/crystal.section.md @@ -4,12 +4,12 @@ This section uses [Mint](https://github.com/mint-lang/mint) as an example for how to build a Crystal package. -If the Crystal project has any dependencies, the first step is to get a `shards.nix` file encoding those. Get a copy of the project and go to its root directory such that its `shard.lock` file is in the current directory, then run `crystal2nix` in it - +If the Crystal project has any dependencies, the first step is to get a `shards.nix` file encoding those. Get a copy of the project and go to its root directory such that its `shard.lock` file is in the current directory. Executable projects should usually commit the `shard.lock` file, but sometimes that's not the case, which means you need to generate it yourself. With an existing `shard.lock` file, `crystal2nix` can be run. ```bash $ git clone https://github.com/mint-lang/mint $ cd mint $ git checkout 0.5.0 +$ if [ ! -f shard.lock ]; then nix-shell -p shards --run "shards lock"; fi $ nix-shell -p crystal2nix --run crystal2nix ``` diff --git a/nixos/modules/module-list.nix b/nixos/modules/module-list.nix index 5610813d9ad0..00d0103e6b79 100644 --- a/nixos/modules/module-list.nix +++ b/nixos/modules/module-list.nix @@ -992,6 +992,7 @@ ./services/web-apps/jitsi-meet.nix ./services/web-apps/keycloak.nix ./services/web-apps/lemmy.nix + ./services/web-apps/invidious.nix ./services/web-apps/limesurvey.nix ./services/web-apps/mastodon.nix ./services/web-apps/mattermost.nix diff --git a/nixos/modules/services/web-apps/invidious.nix b/nixos/modules/services/web-apps/invidious.nix new file mode 100644 index 000000000000..7fb826af5835 --- /dev/null +++ b/nixos/modules/services/web-apps/invidious.nix @@ -0,0 +1,263 @@ +{ lib, config, pkgs, options, ... }: +let + cfg = config.services.invidious; + # To allow injecting secrets with jq, json (instead of yaml) is used + settingsFormat = pkgs.formats.json { }; + inherit (lib) types; + + settingsFile = settingsFormat.generate "invidious-settings" cfg.settings; + + serviceConfig = { + systemd.services.invidious = { + description = "Invidious (An alternative YouTube front-end)"; + wants = [ "network-online.target" ]; + after = [ "syslog.target" "network-online.target" ]; + wantedBy = [ "multi-user.target" ]; + + script = + let + jqFilter = "." + + lib.optionalString (cfg.database.host != null) "[0].db.password = \"'\"'\"$(cat ${lib.escapeShellArg cfg.database.passwordFile})\"'\"'\"" + + " | .[0]" + + lib.optionalString (cfg.extraSettingsFile != null) " * .[1]"; + jqFiles = [ settingsFile ] ++ lib.optional (cfg.extraSettingsFile != null) cfg.extraSettingsFile; + in + '' + export INVIDIOUS_CONFIG="$(${pkgs.jq}/bin/jq -s "${jqFilter}" ${lib.escapeShellArgs jqFiles})" + exec ${cfg.package}/bin/invidious + ''; + + serviceConfig = { + RestartSec = "2s"; + DynamicUser = true; + + CapabilityBoundingSet = ""; + PrivateDevices = true; + PrivateUsers = true; + ProtectHome = true; + ProtectKernelLogs = true; + ProtectProc = "invisible"; + RestrictAddressFamilies = [ "AF_UNIX" "AF_INET" "AF_INET6" ]; + RestrictNamespaces = true; + SystemCallArchitectures = "native"; + SystemCallFilter = [ "@system-service" "~@privileged" "~@resources" ]; + }; + }; + + services.invidious.settings = { + inherit (cfg) port; + + # Automatically initialises and migrates the database if necessary + check_tables = true; + + db = { + user = lib.mkDefault "kemal"; + dbname = lib.mkDefault "invidious"; + port = cfg.database.port; + # Blank for unix sockets, see + # https://github.com/will/crystal-pg/blob/1548bb255210/src/pq/conninfo.cr#L100-L108 + host = if cfg.database.host == null then "" else cfg.database.host; + # Not needed because peer authentication is enabled + password = lib.mkIf (cfg.database.host == null) ""; + }; + } // (lib.optionalAttrs (cfg.domain != null) { + inherit (cfg) domain; + }); + + assertions = [{ + assertion = cfg.database.host != null -> cfg.database.passwordFile != null; + message = "If database host isn't null, database password needs to be set"; + }]; + }; + + # Settings necessary for running with an automatically managed local database + localDatabaseConfig = lib.mkIf cfg.database.createLocally { + # Default to using the local database if we create it + services.invidious.database.host = lib.mkDefault null; + + services.postgresql = { + enable = true; + ensureDatabases = lib.singleton cfg.settings.db.dbname; + ensureUsers = lib.singleton { + name = cfg.settings.db.user; + ensurePermissions = { + "DATABASE ${cfg.settings.db.dbname}" = "ALL PRIVILEGES"; + }; + }; + # This is only needed because the unix user invidious isn't the same as + # the database user. This tells postgres to map one to the other. + identMap = '' + invidious invidious ${cfg.settings.db.user} + ''; + # And this specifically enables peer authentication for only this + # database, which allows passwordless authentication over the postgres + # unix socket for the user map given above. + authentication = '' + local ${cfg.settings.db.dbname} ${cfg.settings.db.user} peer map=invidious + ''; + }; + + systemd.services.invidious-db-clean = { + description = "Invidious database cleanup"; + documentation = [ "https://docs.invidious.io/Database-Information-and-Maintenance.md" ]; + startAt = lib.mkDefault "weekly"; + path = [ config.services.postgresql.package ]; + script = '' + psql ${cfg.settings.db.dbname} ${cfg.settings.db.user} -c "DELETE FROM nonces * WHERE expire < current_timestamp" + psql ${cfg.settings.db.dbname} ${cfg.settings.db.user} -c "TRUNCATE TABLE videos" + ''; + serviceConfig = { + DynamicUser = true; + User = "invidious"; + }; + }; + + systemd.services.invidious = { + requires = [ "postgresql.service" ]; + after = [ "postgresql.service" ]; + + serviceConfig = { + User = "invidious"; + }; + }; + }; + + nginxConfig = lib.mkIf cfg.nginx.enable { + services.invidious.settings = { + https_only = config.services.nginx.virtualHosts.${cfg.domain}.forceSSL; + external_port = 80; + }; + + services.nginx = { + enable = true; + virtualHosts.${cfg.domain} = { + locations."/".proxyPass = "http://127.0.0.1:${toString cfg.port}"; + + enableACME = lib.mkDefault true; + forceSSL = lib.mkDefault true; + }; + }; + + assertions = [{ + assertion = cfg.domain != null; + message = "To use services.invidious.nginx, you need to set services.invidious.domain"; + }]; + }; +in +{ + options.services.invidious = { + enable = lib.mkEnableOption "Invidious"; + + package = lib.mkOption { + type = types.package; + default = pkgs.invidious; + defaultText = "pkgs.invidious"; + description = "The Invidious package to use."; + }; + + settings = lib.mkOption { + type = settingsFormat.type; + default = { }; + description = '' + The settings Invidious should use. + + See config.example.yml for a list of all possible options. + ''; + }; + + extraSettingsFile = lib.mkOption { + type = types.nullOr types.str; + default = null; + description = '' + A file including Invidious settings. + + It gets merged with the setttings specified in + and can be used to store secrets like hmac_key outside of the nix store. + ''; + }; + + # This needs to be outside of settings to avoid infinite recursion + # (determining if nginx should be enabled and therefore the settings + # modified). + domain = lib.mkOption { + type = types.nullOr types.str; + default = null; + description = '' + The FQDN Invidious is reachable on. + + This is used to configure nginx and for building absolute URLs. + ''; + }; + + port = lib.mkOption { + type = types.port; + # Default from https://docs.invidious.io/Configuration.md + default = 3000; + description = '' + The port Invidious should listen on. + + To allow access from outside, + you can use either + or add config.services.invidious.port to . + ''; + }; + + database = { + createLocally = lib.mkOption { + type = types.bool; + default = true; + description = '' + Whether to create a local database with PostgreSQL. + ''; + }; + + host = lib.mkOption { + type = types.nullOr types.str; + default = null; + description = '' + The database host Invidious should use. + + If null, the local unix socket is used. Otherwise + TCP is used. + ''; + }; + + port = lib.mkOption { + type = types.port; + default = options.services.postgresql.port.default; + description = '' + The port of the database Invidious should use. + + Defaults to the the default postgresql port. + ''; + }; + + passwordFile = lib.mkOption { + type = types.nullOr types.str; + apply = lib.mapNullable toString; + default = null; + description = '' + Path to file containing the database password. + ''; + }; + }; + + nginx.enable = lib.mkOption { + type = types.bool; + default = false; + description = '' + Whether to configure nginx as a reverse proxy for Invidious. + + It serves it under the domain specified in with enabled TLS and ACME. + Further configuration can be done through , + which can also be used to disable AMCE and TLS. + ''; + }; + }; + + config = lib.mkIf cfg.enable (lib.mkMerge [ + serviceConfig + localDatabaseConfig + nginxConfig + ]); +} diff --git a/nixos/tests/all-tests.nix b/nixos/tests/all-tests.nix index 906b652e1b8e..b3f6425763ca 100644 --- a/nixos/tests/all-tests.nix +++ b/nixos/tests/all-tests.nix @@ -173,6 +173,7 @@ in hedgedoc = handleTest ./hedgedoc.nix {}; herbstluftwm = handleTest ./herbstluftwm.nix {}; installed-tests = pkgs.recurseIntoAttrs (handleTest ./installed-tests {}); + invidious = handleTest ./invidious.nix {}; oci-containers = handleTestOn ["x86_64-linux"] ./oci-containers.nix {}; # 9pnet_virtio used to mount /nix partition doesn't support # hibernation. This test happens to work on x86_64-linux but diff --git a/nixos/tests/invidious.nix b/nixos/tests/invidious.nix new file mode 100644 index 000000000000..8b831715a441 --- /dev/null +++ b/nixos/tests/invidious.nix @@ -0,0 +1,81 @@ +import ./make-test-python.nix ({ pkgs, ... }: { + name = "invidious"; + + meta = with pkgs.lib.maintainers; { + maintainers = [ sbruder ]; + }; + + machine = { config, lib, pkgs, ... }: { + services.invidious = { + enable = true; + }; + + specialisation = { + nginx.configuration = { + services.invidious = { + nginx.enable = true; + domain = "invidious.example.com"; + }; + services.nginx.virtualHosts."invidious.example.com" = { + forceSSL = false; + enableACME = false; + }; + networking.hosts."127.0.0.1" = [ "invidious.example.com" ]; + }; + postgres-tcp.configuration = { + services.invidious = { + database = { + createLocally = false; + host = "127.0.0.1"; + passwordFile = toString (pkgs.writeText "database-password" "correct horse battery staple"); + }; + }; + # Normally not needed because when connecting to postgres over TCP/IP + # the database is most likely on another host. + systemd.services.invidious = { + after = [ "postgresql.service" ]; + requires = [ "postgresql.service" ]; + }; + services.postgresql = + let + inherit (config.services.invidious.settings.db) dbname user; + in + { + enable = true; + initialScript = pkgs.writeText "init-postgres-with-password" '' + CREATE USER kemal WITH PASSWORD 'correct horse battery staple'; + CREATE DATABASE invidious; + GRANT ALL PRIVILEGES ON DATABASE invidious TO kemal; + ''; + }; + }; + }; + }; + + testScript = { nodes, ... }: '' + def curl_assert_status_code(url, code, form=None): + assert int(machine.succeed(f"curl -s -o /dev/null -w %{{http_code}} {'-F ' + form + ' ' if form else '''}{url}")) == code + + + def activate_specialisation(name: str): + machine.succeed(f"${nodes.machine.config.system.build.toplevel}/specialisation/{name}/bin/switch-to-configuration test >&2") + + + url = "http://localhost:${toString nodes.machine.config.services.invidious.port}" + port = ${toString nodes.machine.config.services.invidious.port} + + machine.wait_for_open_port(port) + curl_assert_status_code(f"{url}/search", 200) + + activate_specialisation("nginx") + machine.wait_for_open_port(80) + curl_assert_status_code("http://invidious.example.com/search", 200) + + # Remove the state so the `initialScript` gets run + machine.succeed("systemctl stop postgresql") + machine.succeed("rm -r /var/lib/postgresql") + activate_specialisation("postgres-tcp") + machine.wait_for_open_port(port) + curl_assert_status_code(f"{url}/search", 200) + ''; +}) diff --git a/pkgs/servers/invidious/default.nix b/pkgs/servers/invidious/default.nix new file mode 100644 index 000000000000..183770a94f41 --- /dev/null +++ b/pkgs/servers/invidious/default.nix @@ -0,0 +1,99 @@ +{ lib, crystal, fetchFromGitHub, librsvg, pkg-config, libxml2, openssl, sqlite, lsquic, nixosTests }: +let + # When updating, always update the following: + # * the git revision + # * the version attribute + # * the source hash (sha256) + # If the shards.lock file changed, also the following: + # * shards.nix (by running `crystal2nix` in invidious’ source tree) + # * If the lsquic.cr dependency changed: lsquic in lsquic.nix (version, sha256) + # * If the lsquic version changed: boringssl' in lsquic.nix (version, sha256) + rev = "21b96a31599e890fe063e3e24cf5f3a995779a69"; +in +crystal.buildCrystalPackage rec { + pname = "invidious"; + version = "unstable-2021-10-15"; + + src = fetchFromGitHub { + owner = "iv-org"; + repo = pname; + inherit rev; + sha256 = "sha256-Rp3YqjHbP6szohlaEpgopFNdLK31yrcHtyKCeVz76CA="; + }; + + postPatch = + let + # Replacing by the value (templates) of the variables ensures that building + # fails if upstream changes the way the metadata is formatted. + branchTemplate = ''{{ "#{`git branch | sed -n '/* /s///p'`.strip}" }}''; + commitTemplate = ''{{ "#{`git rev-list HEAD --max-count=1 --abbrev-commit`.strip}" }}''; + versionTemplate = ''{{ "#{`git log -1 --format=%ci | awk '{print $1}' | sed s/-/./g`.strip}" }}''; + # This always uses the latest commit which invalidates the cache even if + # the assets were not changed + assetCommitTemplate = ''{{ "#{`git rev-list HEAD --max-count=1 --abbrev-commit -- assets`.strip}" }}''; + in + '' + # Use the version metadata from the derivation instead of using git at + # build-time + substituteInPlace src/invidious.cr \ + --replace ${lib.escapeShellArg branchTemplate} '"master"' \ + --replace ${lib.escapeShellArg commitTemplate} '"${lib.substring 0 7 rev}"' \ + --replace ${lib.escapeShellArg versionTemplate} '"${lib.replaceChars ["-"] ["."] (lib.substring 9 10 version)}"' \ + --replace ${lib.escapeShellArg assetCommitTemplate} '"${lib.substring 0 7 rev}"' + + # Patch the assets and locales paths to be absolute + substituteInPlace src/invidious.cr \ + --replace 'public_folder "assets"' 'public_folder "${placeholder "out"}/share/invidious/assets"' + substituteInPlace src/invidious/helpers/i18n.cr \ + --replace 'File.read("locales/' 'File.read("${placeholder "out"}/share/invidious/locales/' + + # Reference sql initialisation/migration scripts by absolute path + substituteInPlace src/invidious/helpers/helpers.cr \ + --replace 'config/sql' '${placeholder "out"}/share/invidious/config/sql' + + substituteInPlace src/invidious/users.cr \ + --replace 'Process.run(%(rsvg-convert' 'Process.run(%(${lib.getBin librsvg}/bin/rsvg-convert' + ''; + + nativeBuildInputs = [ pkg-config ]; + buildInputs = [ libxml2 openssl sqlite ]; + + format = "crystal"; + shardsFile = ./shards.nix; + crystalBinaries.invidious.src = "src/invidious.cr"; + + postConfigure = '' + # lib includes nix store paths which can’t be patched, so the links have to + # be dereferenced first. + cp -rL lib lib2 + rm -r lib + mv lib2 lib + chmod +w -R lib + cp ${lsquic}/lib/liblsquic.a lib/lsquic/src/lsquic/ext + ''; + + postInstall = '' + mkdir -p $out/share/invidious/config + + # Copy static parts + cp -r assets locales $out/share/invidious + cp -r config/sql $out/share/invidious/config + ''; + + # Invidious tries to open config/config.yml and connect to the database, even + # when running --help. This specifies a minimal configuration in an + # environment variable. Even though the database is bogus, --help still + # works. + installCheckPhase = '' + INVIDIOUS_CONFIG="database_url: sqlite3:///dev/null" $out/bin/invidious --help + ''; + + passthru.tests = { inherit (nixosTests) invidious; }; + + meta = with lib; { + description = "An open source alternative front-end to YouTube"; + homepage = "https://invidious.io/"; + license = licenses.agpl3; + maintainers = with maintainers; [ infinisil sbruder ]; + }; +} diff --git a/pkgs/servers/invidious/lsquic.nix b/pkgs/servers/invidious/lsquic.nix new file mode 100644 index 000000000000..35b93e4249f2 --- /dev/null +++ b/pkgs/servers/invidious/lsquic.nix @@ -0,0 +1,58 @@ +{ lib, boringssl, stdenv, fetchgit, fetchFromGitHub, cmake, zlib, perl, libevent }: +let + # lsquic requires a specific boringssl version (noted in its README) + boringssl' = boringssl.overrideAttrs (old: rec { + version = "251b5169fd44345f455438312ec4e18ae07fd58c"; + src = fetchgit { + url = "https://boringssl.googlesource.com/boringssl"; + rev = version; + sha256 = "sha256-EU6T9yQCdOLx98Io8o01rEsgxDFF/Xoy42LgPopD2/A="; + }; + }); +in +stdenv.mkDerivation rec { + pname = "lsquic"; + version = "2.18.1"; + + src = fetchFromGitHub { + owner = "litespeedtech"; + repo = pname; + rev = "v${version}"; + sha256 = "sha256-hG8cUvhbCNeMOsKkaJlgGpzUrIx47E/WhmPIdI5F3qM="; + fetchSubmodules = true; + }; + + nativeBuildInputs = [ cmake perl ]; + buildInputs = [ boringssl' libevent zlib ]; + + cmakeFlags = [ + "-DBORINGSSL_DIR=${boringssl'}" + "-DBORINGSSL_LIB_crypto=${boringssl'}/lib/libcrypto.a" + "-DBORINGSSL_LIB_ssl=${boringssl'}/lib/libssl.a" + "-DZLIB_LIB=${zlib}/lib/libz.so" + ]; + + # adapted from lsquic.cr’s Dockerfile + # (https://github.com/iv-org/lsquic.cr/blob/master/docker/Dockerfile) + installPhase = '' + runHook preInstall + + mkdir combinedlib + cd combinedlib + ar -x ${boringssl'}/lib/libssl.a + ar -x ${boringssl'}/lib/libcrypto.a + ar -x ../src/liblsquic/liblsquic.a + ar rc liblsquic.a *.o + ranlib liblsquic.a + install -D liblsquic.a $out/lib/liblsquic.a + + runHook postInstall + ''; + + meta = with lib; { + description = "A library for QUIC and HTTP/3 (version for Invidious)"; + homepage = "https://github.com/litespeedtech/lsquic"; + maintainers = with maintainers; [ infinisil sbruder ]; + license = with licenses; [ openssl isc mit bsd3 ]; # statically links against boringssl, so has to include its licenses + }; +} diff --git a/pkgs/servers/invidious/shards.nix b/pkgs/servers/invidious/shards.nix new file mode 100644 index 000000000000..778b34496a62 --- /dev/null +++ b/pkgs/servers/invidious/shards.nix @@ -0,0 +1,68 @@ +{ + athena-negotiation = { + owner = "athena-framework"; + repo = "negotiation"; + rev = "v0.1.1"; + sha256 = "1vkk59lqrxb0l8kyzs114i3c18zb2bdiah2xhazkk8q7x6fz4yzk"; + }; + backtracer = { + owner = "sija"; + repo = "backtracer.cr"; + rev = "v1.2.1"; + sha256 = "02r1l7rn2wsljkx495s5s7j04zgn73m2kx0hkzs7620camvlwbqq"; + }; + db = { + owner = "crystal-lang"; + repo = "crystal-db"; + rev = "v0.10.1"; + sha256 = "03c5h14z6h2mxnx949lihnyqjd19hcj38iasdwq9fp95h8cld376"; + }; + exception_page = { + owner = "crystal-loot"; + repo = "exception_page"; + rev = "v0.2.0"; + sha256 = "0nlgnh5iykbr1v2132342k2mz6s2laws6nkgqsqlwhhcr4gb4jcx"; + }; + kemal = { + owner = "kemalcr"; + repo = "kemal"; + rev = "v1.1.0"; + sha256 = "07vlvddy4mba9li2bvskzqzywwq55cyvlgkz13q6dsl4zfgc96ca"; + }; + kilt = { + owner = "jeromegn"; + repo = "kilt"; + rev = "v0.6.1"; + sha256 = "0dpc15y9m8c5l9zdfif6jlf7zmkrlm9w4m2igi5xa22fdjwamwfp"; + }; + lsquic = { + owner = "iv-org"; + repo = "lsquic.cr"; + rev = "v2.18.1-2"; + sha256 = "0bljk0pwbjb813dfwrhgi00w2ai09k868xvak4hfzdkbmpc7id6y"; + }; + pg = { + owner = "will"; + repo = "crystal-pg"; + rev = "v0.24.0"; + sha256 = "07i5bqkv5j6y6f8v5cpqdxc5wzzrvgv3ds24znv4mzv6nc84csn4"; + }; + protodec = { + owner = "iv-org"; + repo = "protodec"; + rev = "v0.1.4"; + sha256 = "15azh9izxqgwpgkpicmivfdz31wkibnwy09rwhxsg0lyc4wf8xj9"; + }; + radix = { + owner = "luislavena"; + repo = "radix"; + rev = "v0.4.1"; + sha256 = "1l08cydkdidq9yyil1wl240hvk41iycv04jrg6nx5mkvzw4z1bzg"; + }; + sqlite3 = { + owner = "crystal-lang"; + repo = "crystal-sqlite3"; + rev = "v0.18.0"; + sha256 = "03nnvpchhq9f9ywsm3pk2rrj4a3figw7xs96zdziwgr5znkz6x93"; + }; +} diff --git a/pkgs/top-level/all-packages.nix b/pkgs/top-level/all-packages.nix index b6ef8590684f..fe53386974cb 100644 --- a/pkgs/top-level/all-packages.nix +++ b/pkgs/top-level/all-packages.nix @@ -6401,6 +6401,11 @@ with pkgs; intermodal = callPackage ../tools/misc/intermodal { }; + invidious = callPackage ../servers/invidious { + # needs a specific version of lsquic + lsquic = callPackage ../servers/invidious/lsquic.nix { }; + }; + invoice2data = callPackage ../tools/text/invoice2data { }; inxi = callPackage ../tools/system/inxi { };