diff --git a/pkgs/by-name/ne/nexusmods-app/package.nix b/pkgs/by-name/ne/nexusmods-app/package.nix index 15f282dc61e9..58688dc64611 100644 --- a/pkgs/by-name/ne/nexusmods-app/package.nix +++ b/pkgs/by-name/ne/nexusmods-app/package.nix @@ -5,7 +5,6 @@ desktop-file-utils, dotnetCorePackages, fetchFromGitHub, - fetchurl, imagemagick, lib, xdg-utils, @@ -35,12 +34,6 @@ buildDotnetModule (finalAttrs: { gameHashes = callPackage ./game-hashes { }; - gamesJson = fetchurl { - name = "games.json"; - url = "https://web.archive.org/web/20251221212501id_/https://data.nexusmods.com/file/nexus-data/games.json"; - hash = "sha256-OVy6b/n6n2/TdzfHCWDOTw2yai87ieki21fEp1IGamY="; - }; - enableParallelBuilding = false; # If the whole solution is published, there seems to be a race condition where @@ -73,9 +66,9 @@ buildDotnetModule (finalAttrs: { substituteInPlace src/NexusMods.Games.FileHashes/NexusMods.Games.FileHashes.csproj \ --replace-fail '$(BaseIntermediateOutputPath)games_hashes_db.zip' "$gameHashes" - # Use a pinned version of the nexus API's games.json data + # Use a vendored version of the nexus API's games.json data substituteInPlace src/NexusMods.Networking.NexusWebApi/NexusMods.Networking.NexusWebApi.csproj \ - --replace-fail '$(BaseIntermediateOutputPath)games.json' "$gamesJson" + --replace-fail '$(BaseIntermediateOutputPath)games.json' ${./vendored/games.json} ${lib.optionalString finalAttrs.doCheck '' # For some reason these tests fail (intermittently?) with a zero timestamp diff --git a/pkgs/by-name/ne/nexusmods-app/update-games-json.sh b/pkgs/by-name/ne/nexusmods-app/update-games-json.sh deleted file mode 100755 index 17e15cc452f8..000000000000 --- a/pkgs/by-name/ne/nexusmods-app/update-games-json.sh +++ /dev/null @@ -1,78 +0,0 @@ -#!/usr/bin/env nix-shell -#! nix-shell -i bash -p bash curl common-updater-scripts - -set -eu -o pipefail - -# Set a default attrpath to allow running this update script directly -export UPDATE_NIX_ATTR_PATH="${UPDATE_NIX_ATTR_PATH:-"nexusmods-app"}" - -save_url=https://web.archive.org/save -json_url=https://data.nexusmods.com/file/nexus-data/games.json -self=$(realpath "$0") -dir=$(dirname "$self") -curl_args=( - "$save_url/$json_url" - - --silent - --fail - --location # follow redirects - --head # we only need headers - --output /dev/null # discard body - --write-out '%{url_effective}' - - # Format --data-urlencode as GET query params - --get - # Instruct IA not to crawl linked content - --data-urlencode capture_all=0 - # Only create a snapshot if older than 1 day - --data-urlencode if_not_archived_within=86400 -) -cd "$dir"/../../../../ - -# Ask the Internet Archive to save a snapshot of `games.json` -# We capture its 'HTTP 302 Found' redirect location -echo "Fetching games.json Internet Archive URL" >&2 -url=$(curl "${curl_args[@]}") - -# Use raw replay mode to avoid rewritten links in the JSON content -url=$( - sed -E 's|^https://web\.archive\.org/web/([0-9]+)/|https://web.archive.org/web/\1id_/|' \ - <<<"$url" -) - -# The query parameters are only used in the snapshot request, -# but remain in the redirected URL. Strip them: -url="${url%%\?*}" - -current_url=$( - nix-instantiate --eval --raw \ - --attr "$UPDATE_NIX_ATTR_PATH.gamesJson.url" -) -if [ "$current_url" = "$url" ]; then - echo "games.json has no changes" >&2 - exit -fi - -echo "Downloading and hashing games.json" >&2 -hash=$( - nix --extra-experimental-features nix-command \ - hash convert --hash-algo sha256 --to sri \ - "$(nix-prefetch-url "$url" --type sha256)" -) - -# `update-source-version` needs the package version, -# even though we're updating the games.json source -package_version=$( - nix-instantiate --eval --raw \ - --attr "$UPDATE_NIX_ATTR_PATH.version" -) - -echo "Updating games.json source" >&2 -update-source-version \ - "$UPDATE_NIX_ATTR_PATH" \ - "$package_version" \ - "$hash" \ - "$url" \ - --ignore-same-version \ - --source-key=gamesJson \ - --file="$dir"/package.nix diff --git a/pkgs/by-name/ne/nexusmods-app/update.sh b/pkgs/by-name/ne/nexusmods-app/update.sh index 05ff0144d6f4..8f9e74304345 100755 --- a/pkgs/by-name/ne/nexusmods-app/update.sh +++ b/pkgs/by-name/ne/nexusmods-app/update.sh @@ -10,28 +10,15 @@ self=$(realpath "$0") dir=$(dirname "$self") cd "$dir"/../../../../ -current_version=$( - nix-instantiate --eval --raw \ - --attr "$UPDATE_NIX_ATTR_PATH.version" -) +# Update vendored files +"$dir"/vendored/update.sh + +# Update game_hashes_db +UPDATE_NIX_ATTR_PATH="$UPDATE_NIX_ATTR_PATH.gameHashes" \ + "$dir"/game-hashes/update.sh url=$( nix-instantiate --eval --raw \ --attr "$UPDATE_NIX_ATTR_PATH.meta.homepage" ) - nix-update --url "$url" - -new_version=$( - nix-instantiate --eval --raw \ - --attr "$UPDATE_NIX_ATTR_PATH.version" -) - -if [ "$current_version" != "$new_version" ]; then - # Update games.json - "$dir"/update-games-json.sh - - # Update game_hashes_db - UPDATE_NIX_ATTR_PATH="$UPDATE_NIX_ATTR_PATH.gameHashes" \ - "$dir"/game-hashes/update.sh -fi diff --git a/pkgs/by-name/ne/nexusmods-app/vendored/README.md b/pkgs/by-name/ne/nexusmods-app/vendored/README.md new file mode 100644 index 000000000000..f92cbf87e634 --- /dev/null +++ b/pkgs/by-name/ne/nexusmods-app/vendored/README.md @@ -0,0 +1,31 @@ +This directory contains a vendored copy of `games.json`, along with tooling to generate it. + +## Purpose + +The games data is fetched at runtime by NexusMods.App, however it is also included at build time for two reasons: + +1. It allows tests to run against real data. +2. It is used as cached data, speeding up the app's initial run. + +It is not vital for the file to contain all games, however ideally it should contain all games _supported_ by this version of NexusMods.App. +That way the initial run's cached data is more useful. + +If this file grows too large, because we are including too many games, we can patch the `csproj` build spec so that `games.json` is not used at build time. +We would also need to patch or disable any tests that rely on it. + +## Generating + +`games.json` is generated automatically by `update.sh`, using data from [nexusmods' API][url] and the games listed in `game-ids.nix`. + +To add a new game to `games.json`: +- Inspect the [nexusmods endpoint][url] to find the game's name and ID +- Add the name and ID to `game-ids.nix` +- Run `update.sh` +- Commit the result + +> [!Note] +> Running `update.sh` may also update the existing games, so you may wish to create two separate commits using `git add --patch`. +> One for updating the existing data and another for adding the new game. + +[url]: https://data.nexusmods.com/file/nexus-data/games.json + diff --git a/pkgs/by-name/ne/nexusmods-app/vendored/game-ids.nix b/pkgs/by-name/ne/nexusmods-app/vendored/game-ids.nix new file mode 100644 index 000000000000..764658cd596d --- /dev/null +++ b/pkgs/by-name/ne/nexusmods-app/vendored/game-ids.nix @@ -0,0 +1,12 @@ +# This file lists games to be included in the vendored games.json file. +# It is not critical to include all games, other than those referenced by the test suite. +# Ideally, all games supported by the app will be included, as this can improve first-run performance. +{ + # keep-sorted start case=no numeric=yes + "Baldur's Gate 3" = 3474; + "Cyberpunk 2077" = 3333; + "Mount & Blade II: Bannerlord" = 3174; + "Skyrim Special Edition" = 1704; + "Stardew Valley" = 1303; + # keep-sorted end +} diff --git a/pkgs/by-name/ne/nexusmods-app/vendored/games.json b/pkgs/by-name/ne/nexusmods-app/vendored/games.json new file mode 100644 index 000000000000..6d7b4767e012 --- /dev/null +++ b/pkgs/by-name/ne/nexusmods-app/vendored/games.json @@ -0,0 +1,72 @@ +[ + { + "id": 1303, + "name": "Stardew Valley", + "name_lower": "stardew valley", + "forum_url": "https://forums.nexusmods.com/games/19-stardew-valley/", + "nexusmods_url": "https://www.nexusmods.com/stardewvalley", + "genre": "Simulation", + "file_count": 145440, + "downloads": 642501970, + "domain_name": "stardewvalley", + "approved_date": 1457432329, + "mods": 26417, + "collections": 1958 + }, + { + "id": 1704, + "name": "Skyrim Special Edition", + "name_lower": "skyrim special edition", + "forum_url": "https://forums.nexusmods.com/games/6-skyrim/", + "nexusmods_url": "https://www.nexusmods.com/skyrimspecialedition", + "genre": "RPG", + "file_count": 671705, + "downloads": 9497429062, + "domain_name": "skyrimspecialedition", + "approved_date": 1477480498, + "mods": 120722, + "collections": 4886 + }, + { + "id": 3174, + "name": "Mount & Blade II: Bannerlord", + "name_lower": "mount & blade ii: bannerlord", + "forum_url": "https://forums.nexusmods.com/games/9-mount-blade-ii-bannerlord/", + "nexusmods_url": "https://www.nexusmods.com/mountandblade2bannerlord", + "genre": "Strategy", + "file_count": 51310, + "downloads": 118459993, + "domain_name": "mountandblade2bannerlord", + "approved_date": 1582898627, + "mods": 6455, + "collections": 294 + }, + { + "id": 3333, + "name": "Cyberpunk 2077", + "name_lower": "cyberpunk 2077", + "forum_url": "https://forums.nexusmods.com/games/1-cyberpunk-2077/", + "nexusmods_url": "https://www.nexusmods.com/cyberpunk2077", + "genre": "Action", + "file_count": 127687, + "downloads": 981788054, + "domain_name": "cyberpunk2077", + "approved_date": 1607433331, + "mods": 18464, + "collections": 1613 + }, + { + "id": 3474, + "name": "Baldur's Gate 3", + "name_lower": "baldur's gate 3", + "forum_url": "https://forums.nexusmods.com/games/2-baldurs-gate-3/", + "nexusmods_url": "https://www.nexusmods.com/baldursgate3", + "genre": "RPG", + "file_count": 107598, + "downloads": 366648874, + "domain_name": "baldursgate3", + "approved_date": 1602863114, + "mods": 15345, + "collections": 1725 + } +] diff --git a/pkgs/by-name/ne/nexusmods-app/vendored/update.sh b/pkgs/by-name/ne/nexusmods-app/vendored/update.sh new file mode 100755 index 000000000000..ac44e25fae48 --- /dev/null +++ b/pkgs/by-name/ne/nexusmods-app/vendored/update.sh @@ -0,0 +1,53 @@ +#!/usr/bin/env nix-shell +#! nix-shell -i bash -p bash curl jq + +set -eu -o pipefail + +url='https://data.nexusmods.com/file/nexus-data/games.json' +self=$(realpath "$0") +dir=$(dirname "$self") +tmp=$(mktemp) + +cd "$dir"/../../../../../ + +ids=$( + nix-instantiate --eval --json \ + --argstr file "$dir"/game-ids.nix \ + --expr '{file}: builtins.attrValues (import file)' +) + +echo "Fetching games data" >&2 +curl "$url" \ + --silent \ + --show-error \ + --location | + jq --argjson ids "$ids" \ + 'map(select( .id | IN($ids[]) )) | sort_by(.id)' \ + >"$tmp" + +echo "Validating result" >&2 +nix-instantiate --eval --strict \ + --argstr idsNix "$dir"/game-ids.nix \ + --argstr gamesJson "$tmp" \ + --expr ' + { + idsNix, + gamesJson, + lib ? import , + }: + let + ids = import idsNix; + games = lib.importJSON gamesJson; + in + lib.forEach games ( + { id, name, ... }: + lib.throwIfNot + (id == ids.${name}) + "${name}: id ${toString id} does not match ${toString ids.${name}}" + null + ) + ' \ + >/dev/null + +echo "Installing games.json to $dir" >&2 +mv --force "$tmp" "$dir"/games.json