From 4f021e14a31eded4a7e3076ba7f71b4f0f3d46cb Mon Sep 17 00:00:00 2001 From: Danilo Reyes Date: Fri, 26 Apr 2024 20:16:54 -0600 Subject: [PATCH 1/7] shiori: 1.5.5 -> 1.7.0 --- nixos/modules/services/web-apps/shiori.nix | 76 ++++++++---- nixos/tests/shiori.nix | 133 ++++++++++----------- pkgs/servers/web-apps/shiori/default.nix | 22 ++-- 3 files changed, 135 insertions(+), 96 deletions(-) diff --git a/nixos/modules/services/web-apps/shiori.nix b/nixos/modules/services/web-apps/shiori.nix index 022bb5e43881..208b5432ef8a 100644 --- a/nixos/modules/services/web-apps/shiori.nix +++ b/nixos/modules/services/web-apps/shiori.nix @@ -1,17 +1,15 @@ { config, lib, pkgs, ... }: -with lib; -let - cfg = config.services.shiori; +let cfg = config.services.shiori; in { options = { services.shiori = { - enable = mkEnableOption "Shiori simple bookmarks manager"; + enable = lib.mkEnableOption "Shiori simple bookmarks manager"; - package = mkPackageOption pkgs "shiori" { }; + package = lib.mkPackageOption pkgs "shiori" { }; - address = mkOption { - type = types.str; + address = lib.mkOption { + type = lib.types.str; default = ""; description = '' The IP address on which Shiori will listen. @@ -19,30 +17,55 @@ in { ''; }; - port = mkOption { - type = types.port; + port = lib.mkOption { + type = lib.types.port; default = 8080; description = "The port of the Shiori web application"; }; - webRoot = mkOption { - type = types.str; + webRoot = lib.mkOption { + type = lib.types.str; default = "/"; example = "/shiori"; description = "The root of the Shiori web application"; }; + + environmentFile = lib.mkOption { + type = lib.types.null or lib.types.path; + default = null; + example = "/path/to/environmentFile"; + description = '' + Path to file containing environment variables. + Useful for passing down secrets. + + ''; + }; + + databaseUrl = lib.mkOption { + type = lib.types.null or lib.types.str; + default = null; + example = "postgresql:///shiori?host=/run/postgresql"; + description = "The connection URL to connect to MySQL or PostgreSQL"; + }; }; }; - config = mkIf cfg.enable { - systemd.services.shiori = with cfg; { + config = lib.mkIf cfg.enable { + systemd.services.shiori = { description = "Shiori simple bookmarks manager"; wantedBy = [ "multi-user.target" ]; - - environment.SHIORI_DIR = "/var/lib/shiori"; + after = [ "postgresql.service" "mysql.service" ]; + environment = { + SHIORI_DIR = "/var/lib/shiori"; + } // lib.optionalAttrs (cfg.databaseUrl != null) { + SHIORI_DATABASE_URL = cfg.databaseUrl; + }; serviceConfig = { - ExecStart = "${package}/bin/shiori serve --address '${address}' --port '${toString port}' --webroot '${webRoot}'"; + ExecStart = + "${cfg.package}/bin/shiori server --address '${cfg.address}' --port '${ + toString cfg.port + }' --webroot '${cfg.webRoot}'"; DynamicUser = true; StateDirectory = "shiori"; @@ -50,15 +73,20 @@ in { RuntimeDirectory = "shiori"; # Security options - + EnvironmentFile = + lib.optional (cfg.environmentFile != null) cfg.environmentFile; BindReadOnlyPaths = [ "/nix/store" # For SSL certificates, and the resolv.conf "/etc" - ]; + ] ++ lib.optional (lib.strings.hasInfix "postgres" cfg.databaseUrl + && config.services.postgresql.enable) "/run/postgresql" + ++ lib.optional (lib.strings.hasInfix "mysql" cfg.databaseUrl + && config.services.mysql.enable) "/var/run/mysqld"; CapabilityBoundingSet = ""; + AmbientCapabilities = "CAP_NET_BIND_SERVICE"; DeviceAllow = ""; @@ -78,7 +106,7 @@ in { ProtectKernelTunables = true; RestrictNamespaces = true; - RestrictAddressFamilies = [ "AF_INET" "AF_INET6" ]; + RestrictAddressFamilies = [ "AF_INET" "AF_INET6" "AF_UNIX" ]; RestrictRealtime = true; RestrictSUIDSGID = true; @@ -88,11 +116,17 @@ in { SystemCallErrorNumber = "EPERM"; SystemCallFilter = [ "@system-service" - "~@cpu-emulation" "~@debug" "~@keyring" "~@memlock" "~@obsolete" "~@privileged" "~@setuid" + "~@cpu-emulation" + "~@debug" + "~@keyring" + "~@memlock" + "~@obsolete" + "~@privileged" + "~@setuid" ]; }; }; }; - meta.maintainers = with maintainers; [ minijackson ]; + meta.maintainers = with lib.maintainers; [ minijackson CaptainJawZ ]; } diff --git a/nixos/tests/shiori.nix b/nixos/tests/shiori.nix index d0f68b903f8c..99c1301cc3f8 100644 --- a/nixos/tests/shiori.nix +++ b/nixos/tests/shiori.nix @@ -1,80 +1,79 @@ -import ./make-test-python.nix ({ pkgs, lib, ...}: +import ./make-test-python.nix ({ pkgs, lib, ... }: -{ - name = "shiori"; - meta.maintainers = with lib.maintainers; [ minijackson ]; + { + name = "shiori"; + meta.maintainers = with lib.maintainers; [ minijackson ]; - nodes.machine = - { ... }: - { services.shiori.enable = true; }; + nodes.machine = { ... }: { services.shiori.enable = true; }; - testScript = let - authJSON = pkgs.writeText "auth.json" (builtins.toJSON { - username = "shiori"; - password = "gopher"; - owner = true; - }); + testScript = let + authJSON = pkgs.writeText "auth.json" (builtins.toJSON { + username = "shiori"; + password = "gopher"; + owner = true; + }); - insertBookmark = { - url = "http://example.org"; - title = "Example Bookmark"; - }; + insertBookmark = { + url = "http://example.org"; + title = "Example Bookmark"; + }; - insertBookmarkJSON = pkgs.writeText "insertBookmark.json" (builtins.toJSON insertBookmark); - in '' - import json + insertBookmarkJSON = + pkgs.writeText "insertBookmark.json" (builtins.toJSON insertBookmark); + in '' + import json - machine.wait_for_unit("shiori.service") - machine.wait_for_open_port(8080) - machine.succeed("curl --fail http://localhost:8080/") - machine.succeed("curl --fail --location http://localhost:8080/ | grep -i shiori") + machine.wait_for_unit("shiori.service") + machine.wait_for_open_port(8080) + machine.succeed("curl --fail http://localhost:8080/") + machine.succeed("curl --fail --location http://localhost:8080/ | grep -i shiori") - with subtest("login"): - auth_json = machine.succeed( - "curl --fail --location http://localhost:8080/api/login " - "-X POST -H 'Content-Type:application/json' -d @${authJSON}" - ) - auth_ret = json.loads(auth_json) - session_id = auth_ret["session"] + with subtest("login"): + auth_json = machine.succeed( + "curl --fail --location http://localhost:8080/api/login " + "-X POST -H 'Content-Type:application/json' -d @${authJSON}" + ) + auth_ret = json.loads(auth_json) + session_id = auth_ret["session"] - with subtest("bookmarks"): - with subtest("first use no bookmarks"): - bookmarks_json = machine.succeed( - ( - "curl --fail --location http://localhost:8080/api/bookmarks " - "-H 'X-Session-Id:{}'" - ).format(session_id) - ) + with subtest("bookmarks"): + with subtest("first use no bookmarks"): + bookmarks_json = machine.succeed( + ( + "curl --fail --location http://localhost:8080/api/bookmarks " + "-H 'X-Session-Id:{}'" + ).format(session_id) + ) - if json.loads(bookmarks_json)["bookmarks"] != []: - raise Exception("Shiori have a bookmark on first use") + if json.loads(bookmarks_json)["bookmarks"] != []: + raise Exception("Shiori have a bookmark on first use") - with subtest("insert bookmark"): - machine.succeed( - ( - "curl --fail --location http://localhost:8080/api/bookmarks " - "-X POST -H 'X-Session-Id:{}' " - "-H 'Content-Type:application/json' -d @${insertBookmarkJSON}" - ).format(session_id) - ) + with subtest("insert bookmark"): + machine.succeed( + ( + "curl --fail --location http://localhost:8080/api/bookmarks " + "-X POST -H 'X-Session-Id:{}' " + "-H 'Content-Type:application/json' -d @${insertBookmarkJSON}" + ).format(session_id) + ) - with subtest("get inserted bookmark"): - bookmarks_json = machine.succeed( - ( - "curl --fail --location http://localhost:8080/api/bookmarks " - "-H 'X-Session-Id:{}'" - ).format(session_id) - ) + with subtest("get inserted bookmark"): + bookmarks_json = machine.succeed( + ( + "curl --fail --location http://localhost:8080/api/bookmarks " + "-H 'X-Session-Id:{}'" + ).format(session_id) + ) - bookmarks = json.loads(bookmarks_json)["bookmarks"] - if len(bookmarks) != 1: - raise Exception("Shiori didn't save the bookmark") + bookmarks = json.loads(bookmarks_json)["bookmarks"] + if len(bookmarks) != 1: + raise Exception("Shiori didn't save the bookmark") - bookmark = bookmarks[0] - if ( - bookmark["url"] != "${insertBookmark.url}" - or bookmark["title"] != "${insertBookmark.title}" - ): - raise Exception("Inserted bookmark doesn't have same URL or title") - ''; -}) + bookmark = bookmarks[0] + if ( + bookmark["url"] != "${insertBookmark.url}" + or bookmark["title"] != "${insertBookmark.title}" + ): + raise Exception("Inserted bookmark doesn't have same URL or title") + ''; + }) diff --git a/pkgs/servers/web-apps/shiori/default.nix b/pkgs/servers/web-apps/shiori/default.nix index 8a9fc7973f98..bd6a22287e5f 100644 --- a/pkgs/servers/web-apps/shiori/default.nix +++ b/pkgs/servers/web-apps/shiori/default.nix @@ -1,10 +1,10 @@ -{ lib, buildGoModule, fetchFromGitHub, nixosTests }: +{ lib, buildGoModule, fetchFromGitHub, nixosTests, installShellFiles }: buildGoModule rec { pname = "shiori"; - version = "1.5.5"; + version = "1.7.0"; - vendorHash = "sha256-suWdtqf5IZntEVD+NHGD6RsL1tjcGH9vh5skISW+aCc="; + vendorHash = "sha256-fakRqgoEcdzw9WZuubaxfGfvVrMvb8gV/IwPikMnfRQ="; doCheck = false; @@ -12,18 +12,24 @@ buildGoModule rec { owner = "go-shiori"; repo = pname; rev = "v${version}"; - sha256 = "sha256-kGPvCYvLLixEH9qih/F3StUyGPqlKukTWLSw41+Mq8E="; + sha256 = "sha256-5+hTtvBnj3Nh5HitReVkLift9LTiMYVuuYx5EirN0SA="; }; - passthru.tests = { - smoke-test = nixosTests.shiori; - }; + nativeBuildInputs = [ installShellFiles ]; + postInstall = '' + installShellCompletion --cmd shiori \ + --bash <($out/bin/shiori completion bash) \ + --fish <($out/bin/shiori completion fish) \ + --zsh <($out/bin/shiori completion zsh) + ''; + + # passthru.tests.smoke-test = nixosTests.shiori; # test broken meta = with lib; { description = "Simple bookmark manager built with Go"; mainProgram = "shiori"; homepage = "https://github.com/go-shiori/shiori"; license = licenses.mit; - maintainers = with maintainers; [ minijackson ]; + maintainers = with maintainers; [ minijackson CaptainJawZ ]; }; } From 0b7f2a2cdae403b07d920e518519ddc2cdcae263 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Sun, 30 Jun 2024 12:56:47 +0200 Subject: [PATCH 2/7] nixos/shiori: comment out non-working tests --- nixos/tests/shiori.nix | 86 ++++++++++++------------ pkgs/servers/web-apps/shiori/default.nix | 2 +- 2 files changed, 45 insertions(+), 43 deletions(-) diff --git a/nixos/tests/shiori.nix b/nixos/tests/shiori.nix index 99c1301cc3f8..ba9b42235df2 100644 --- a/nixos/tests/shiori.nix +++ b/nixos/tests/shiori.nix @@ -21,59 +21,61 @@ import ./make-test-python.nix ({ pkgs, lib, ... }: insertBookmarkJSON = pkgs.writeText "insertBookmark.json" (builtins.toJSON insertBookmark); in '' - import json + #import json machine.wait_for_unit("shiori.service") machine.wait_for_open_port(8080) machine.succeed("curl --fail http://localhost:8080/") machine.succeed("curl --fail --location http://localhost:8080/ | grep -i shiori") - with subtest("login"): - auth_json = machine.succeed( - "curl --fail --location http://localhost:8080/api/login " - "-X POST -H 'Content-Type:application/json' -d @${authJSON}" - ) - auth_ret = json.loads(auth_json) - session_id = auth_ret["session"] + # The test code below no longer works because the API authentication has changed. - with subtest("bookmarks"): - with subtest("first use no bookmarks"): - bookmarks_json = machine.succeed( - ( - "curl --fail --location http://localhost:8080/api/bookmarks " - "-H 'X-Session-Id:{}'" - ).format(session_id) - ) + #with subtest("login"): + # auth_json = machine.succeed( + # "curl --fail --location http://localhost:8080/api/login " + # "-X POST -H 'Content-Type:application/json' -d @${authJSON}" + # ) + # auth_ret = json.loads(auth_json) + # session_id = auth_ret["session"] - if json.loads(bookmarks_json)["bookmarks"] != []: - raise Exception("Shiori have a bookmark on first use") + #with subtest("bookmarks"): + # with subtest("first use no bookmarks"): + # bookmarks_json = machine.succeed( + # ( + # "curl --fail --location http://localhost:8080/api/bookmarks " + # "-H 'X-Session-Id:{}'" + # ).format(session_id) + # ) - with subtest("insert bookmark"): - machine.succeed( - ( - "curl --fail --location http://localhost:8080/api/bookmarks " - "-X POST -H 'X-Session-Id:{}' " - "-H 'Content-Type:application/json' -d @${insertBookmarkJSON}" - ).format(session_id) - ) + # if json.loads(bookmarks_json)["bookmarks"] != []: + # raise Exception("Shiori have a bookmark on first use") - with subtest("get inserted bookmark"): - bookmarks_json = machine.succeed( - ( - "curl --fail --location http://localhost:8080/api/bookmarks " - "-H 'X-Session-Id:{}'" - ).format(session_id) - ) + # with subtest("insert bookmark"): + # machine.succeed( + # ( + # "curl --fail --location http://localhost:8080/api/bookmarks " + # "-X POST -H 'X-Session-Id:{}' " + # "-H 'Content-Type:application/json' -d @${insertBookmarkJSON}" + # ).format(session_id) + # ) - bookmarks = json.loads(bookmarks_json)["bookmarks"] - if len(bookmarks) != 1: - raise Exception("Shiori didn't save the bookmark") + # with subtest("get inserted bookmark"): + # bookmarks_json = machine.succeed( + # ( + # "curl --fail --location http://localhost:8080/api/bookmarks " + # "-H 'X-Session-Id:{}'" + # ).format(session_id) + # ) - bookmark = bookmarks[0] - if ( - bookmark["url"] != "${insertBookmark.url}" - or bookmark["title"] != "${insertBookmark.title}" - ): - raise Exception("Inserted bookmark doesn't have same URL or title") + # bookmarks = json.loads(bookmarks_json)["bookmarks"] + # if len(bookmarks) != 1: + # raise Exception("Shiori didn't save the bookmark") + + # bookmark = bookmarks[0] + # if ( + # bookmark["url"] != "${insertBookmark.url}" + # or bookmark["title"] != "${insertBookmark.title}" + # ): + # raise Exception("Inserted bookmark doesn't have same URL or title") ''; }) diff --git a/pkgs/servers/web-apps/shiori/default.nix b/pkgs/servers/web-apps/shiori/default.nix index bd6a22287e5f..61c64b0af5dc 100644 --- a/pkgs/servers/web-apps/shiori/default.nix +++ b/pkgs/servers/web-apps/shiori/default.nix @@ -23,7 +23,7 @@ buildGoModule rec { --zsh <($out/bin/shiori completion zsh) ''; - # passthru.tests.smoke-test = nixosTests.shiori; # test broken + passthru.tests.smoke-test = nixosTests.shiori; meta = with lib; { description = "Simple bookmark manager built with Go"; From 992f8bf968f23b026ed5355332d3b2434ea5515b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Sun, 30 Jun 2024 12:57:31 +0200 Subject: [PATCH 3/7] nixos/shiori: fix nixos types for paths --- nixos/modules/services/web-apps/shiori.nix | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nixos/modules/services/web-apps/shiori.nix b/nixos/modules/services/web-apps/shiori.nix index 208b5432ef8a..53662010070c 100644 --- a/nixos/modules/services/web-apps/shiori.nix +++ b/nixos/modules/services/web-apps/shiori.nix @@ -31,7 +31,7 @@ in { }; environmentFile = lib.mkOption { - type = lib.types.null or lib.types.path; + type = lib.types.nullOr lib.types.path; default = null; example = "/path/to/environmentFile"; description = '' @@ -42,7 +42,7 @@ in { }; databaseUrl = lib.mkOption { - type = lib.types.null or lib.types.str; + type = lib.types.nullOr lib.types.str; default = null; example = "postgresql:///shiori?host=/run/postgresql"; description = "The connection URL to connect to MySQL or PostgreSQL"; From 84a8354c5809e41b1e57d1537bb23e7378e6ad1a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Sun, 30 Jun 2024 13:03:54 +0200 Subject: [PATCH 4/7] nixos/shiori: fix BindPaths conditions if databaseUrl is not set --- nixos/modules/services/web-apps/shiori.nix | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/nixos/modules/services/web-apps/shiori.nix b/nixos/modules/services/web-apps/shiori.nix index 53662010070c..2dc76ca00fde 100644 --- a/nixos/modules/services/web-apps/shiori.nix +++ b/nixos/modules/services/web-apps/shiori.nix @@ -80,10 +80,14 @@ in { # For SSL certificates, and the resolv.conf "/etc" - ] ++ lib.optional (lib.strings.hasInfix "postgres" cfg.databaseUrl - && config.services.postgresql.enable) "/run/postgresql" - ++ lib.optional (lib.strings.hasInfix "mysql" cfg.databaseUrl - && config.services.mysql.enable) "/var/run/mysqld"; + ] ++ lib.optional (config.services.postgresql.enable && + cfg.databaseUrl != null && + lib.strings.hasPrefix "postgres://" cfg.databaseUrl) + "/run/postgresql" + ++ lib.optional (config.services.mysql.enable && + cfg.databaseUrl != null && + lib.strings.hasPrefix "mysql://" cfg.databaseUrl) + "/var/run/mysqld"; CapabilityBoundingSet = ""; AmbientCapabilities = "CAP_NET_BIND_SERVICE"; From 3ac2687c48671f8ed0288c71afa299be9f246f86 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Sun, 30 Jun 2024 13:07:50 +0200 Subject: [PATCH 5/7] shiori: fix cross-compilation --- pkgs/servers/web-apps/shiori/default.nix | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkgs/servers/web-apps/shiori/default.nix b/pkgs/servers/web-apps/shiori/default.nix index 61c64b0af5dc..98b084ad50e3 100644 --- a/pkgs/servers/web-apps/shiori/default.nix +++ b/pkgs/servers/web-apps/shiori/default.nix @@ -1,4 +1,4 @@ -{ lib, buildGoModule, fetchFromGitHub, nixosTests, installShellFiles }: +{ lib, buildGoModule, fetchFromGitHub, nixosTests, installShellFiles, stdenv }: buildGoModule rec { pname = "shiori"; @@ -16,7 +16,7 @@ buildGoModule rec { }; nativeBuildInputs = [ installShellFiles ]; - postInstall = '' + postInstall = lib.optionalString (stdenv.hostPlatform != stdenv.buildPlatform) '' installShellCompletion --cmd shiori \ --bash <($out/bin/shiori completion bash) \ --fish <($out/bin/shiori completion fish) \ From d6cf181343c027988370693c624d220302759eb7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Sun, 30 Jun 2024 13:41:06 +0200 Subject: [PATCH 6/7] nixos/shiori: fix databaseUrl example --- nixos/modules/services/web-apps/shiori.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nixos/modules/services/web-apps/shiori.nix b/nixos/modules/services/web-apps/shiori.nix index 2dc76ca00fde..df3eeaef1618 100644 --- a/nixos/modules/services/web-apps/shiori.nix +++ b/nixos/modules/services/web-apps/shiori.nix @@ -44,7 +44,7 @@ in { databaseUrl = lib.mkOption { type = lib.types.nullOr lib.types.str; default = null; - example = "postgresql:///shiori?host=/run/postgresql"; + example = "postgres:///shiori?host=/run/postgresql"; description = "The connection URL to connect to MySQL or PostgreSQL"; }; }; From 64bd039d92194b38421cf06c858bec22fe580620 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Sun, 30 Jun 2024 13:48:21 +0200 Subject: [PATCH 7/7] rl-2411: document new setting required for shiori --- nixos/doc/manual/release-notes/rl-2411.section.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/nixos/doc/manual/release-notes/rl-2411.section.md b/nixos/doc/manual/release-notes/rl-2411.section.md index 08e32ebe7d98..6a82ee946677 100644 --- a/nixos/doc/manual/release-notes/rl-2411.section.md +++ b/nixos/doc/manual/release-notes/rl-2411.section.md @@ -102,6 +102,14 @@ for `stateVersion` ≥ 24.11. (It was previously using SQLite for structured data and the filesystem for blobs). +- The `shiori` service now requires an HTTP secret value `SHIORI_HTTP_SECRET_KEY` to be provided via environment variable. The nixos module therefore, now provides an environmentFile option: + + ``` + # This is how a environment file can be generated: + # $ printf "SHIORI_HTTP_SECRET_KEY=%s\n" "$(openssl rand -hex 16)" > /path/to/env-file + services.shiori.environmentFile = "/path/to/env-file"; + ``` + - `libe57format` has been updated to `>= 3.0.0`, which contains some backward-incompatible API changes. See the [release note](https://github.com/asmaloney/libE57Format/releases/tag/v3.0.0) for more details. - `gitlab` deprecated support for *runner registration tokens* in GitLab 16.0, disabled their support in GitLab 17.0 and will