From eb09752753b80cc5caeb1f5be0e07b8a7ee6be8c Mon Sep 17 00:00:00 2001 From: Matt Sturgeon Date: Sun, 21 Dec 2025 19:03:19 +0000 Subject: [PATCH] nexusmods-app: fetch games.json from the Internet Archive Instead of vendoring a subset of `games.json` in-tree, fetch a pinned copy from the Internet Archive's Wayback Machine. Rewrote the update script to ask the Internet Archive to save a snapshot of `games.json`. --- pkgs/by-name/ne/nexusmods-app/package.nix | 11 ++- .../ne/nexusmods-app/update-games-json.sh | 78 +++++++++++++++++++ pkgs/by-name/ne/nexusmods-app/update.sh | 4 +- .../ne/nexusmods-app/vendored/README.md | 31 -------- .../ne/nexusmods-app/vendored/game-ids.nix | 12 --- .../ne/nexusmods-app/vendored/games.json | 72 ----------------- .../ne/nexusmods-app/vendored/update.sh | 53 ------------- 7 files changed, 89 insertions(+), 172 deletions(-) create mode 100755 pkgs/by-name/ne/nexusmods-app/update-games-json.sh delete mode 100644 pkgs/by-name/ne/nexusmods-app/vendored/README.md delete mode 100644 pkgs/by-name/ne/nexusmods-app/vendored/game-ids.nix delete mode 100644 pkgs/by-name/ne/nexusmods-app/vendored/games.json delete mode 100755 pkgs/by-name/ne/nexusmods-app/vendored/update.sh diff --git a/pkgs/by-name/ne/nexusmods-app/package.nix b/pkgs/by-name/ne/nexusmods-app/package.nix index 58688dc64611..15f282dc61e9 100644 --- a/pkgs/by-name/ne/nexusmods-app/package.nix +++ b/pkgs/by-name/ne/nexusmods-app/package.nix @@ -5,6 +5,7 @@ desktop-file-utils, dotnetCorePackages, fetchFromGitHub, + fetchurl, imagemagick, lib, xdg-utils, @@ -34,6 +35,12 @@ buildDotnetModule (finalAttrs: { gameHashes = callPackage ./game-hashes { }; + gamesJson = fetchurl { + name = "games.json"; + url = "https://web.archive.org/web/20251221212501id_/https://data.nexusmods.com/file/nexus-data/games.json"; + hash = "sha256-OVy6b/n6n2/TdzfHCWDOTw2yai87ieki21fEp1IGamY="; + }; + enableParallelBuilding = false; # If the whole solution is published, there seems to be a race condition where @@ -66,9 +73,9 @@ buildDotnetModule (finalAttrs: { substituteInPlace src/NexusMods.Games.FileHashes/NexusMods.Games.FileHashes.csproj \ --replace-fail '$(BaseIntermediateOutputPath)games_hashes_db.zip' "$gameHashes" - # Use a vendored version of the nexus API's games.json data + # Use a pinned version of the nexus API's games.json data substituteInPlace src/NexusMods.Networking.NexusWebApi/NexusMods.Networking.NexusWebApi.csproj \ - --replace-fail '$(BaseIntermediateOutputPath)games.json' ${./vendored/games.json} + --replace-fail '$(BaseIntermediateOutputPath)games.json' "$gamesJson" ${lib.optionalString finalAttrs.doCheck '' # For some reason these tests fail (intermittently?) with a zero timestamp diff --git a/pkgs/by-name/ne/nexusmods-app/update-games-json.sh b/pkgs/by-name/ne/nexusmods-app/update-games-json.sh new file mode 100755 index 000000000000..17e15cc452f8 --- /dev/null +++ b/pkgs/by-name/ne/nexusmods-app/update-games-json.sh @@ -0,0 +1,78 @@ +#!/usr/bin/env nix-shell +#! nix-shell -i bash -p bash curl common-updater-scripts + +set -eu -o pipefail + +# Set a default attrpath to allow running this update script directly +export UPDATE_NIX_ATTR_PATH="${UPDATE_NIX_ATTR_PATH:-"nexusmods-app"}" + +save_url=https://web.archive.org/save +json_url=https://data.nexusmods.com/file/nexus-data/games.json +self=$(realpath "$0") +dir=$(dirname "$self") +curl_args=( + "$save_url/$json_url" + + --silent + --fail + --location # follow redirects + --head # we only need headers + --output /dev/null # discard body + --write-out '%{url_effective}' + + # Format --data-urlencode as GET query params + --get + # Instruct IA not to crawl linked content + --data-urlencode capture_all=0 + # Only create a snapshot if older than 1 day + --data-urlencode if_not_archived_within=86400 +) +cd "$dir"/../../../../ + +# Ask the Internet Archive to save a snapshot of `games.json` +# We capture its 'HTTP 302 Found' redirect location +echo "Fetching games.json Internet Archive URL" >&2 +url=$(curl "${curl_args[@]}") + +# Use raw replay mode to avoid rewritten links in the JSON content +url=$( + sed -E 's|^https://web\.archive\.org/web/([0-9]+)/|https://web.archive.org/web/\1id_/|' \ + <<<"$url" +) + +# The query parameters are only used in the snapshot request, +# but remain in the redirected URL. Strip them: +url="${url%%\?*}" + +current_url=$( + nix-instantiate --eval --raw \ + --attr "$UPDATE_NIX_ATTR_PATH.gamesJson.url" +) +if [ "$current_url" = "$url" ]; then + echo "games.json has no changes" >&2 + exit +fi + +echo "Downloading and hashing games.json" >&2 +hash=$( + nix --extra-experimental-features nix-command \ + hash convert --hash-algo sha256 --to sri \ + "$(nix-prefetch-url "$url" --type sha256)" +) + +# `update-source-version` needs the package version, +# even though we're updating the games.json source +package_version=$( + nix-instantiate --eval --raw \ + --attr "$UPDATE_NIX_ATTR_PATH.version" +) + +echo "Updating games.json source" >&2 +update-source-version \ + "$UPDATE_NIX_ATTR_PATH" \ + "$package_version" \ + "$hash" \ + "$url" \ + --ignore-same-version \ + --source-key=gamesJson \ + --file="$dir"/package.nix diff --git a/pkgs/by-name/ne/nexusmods-app/update.sh b/pkgs/by-name/ne/nexusmods-app/update.sh index 6606411cf739..05ff0144d6f4 100755 --- a/pkgs/by-name/ne/nexusmods-app/update.sh +++ b/pkgs/by-name/ne/nexusmods-app/update.sh @@ -28,8 +28,8 @@ new_version=$( ) if [ "$current_version" != "$new_version" ]; then - # Update vendored files - "$dir"/vendored/update.sh + # Update games.json + "$dir"/update-games-json.sh # Update game_hashes_db UPDATE_NIX_ATTR_PATH="$UPDATE_NIX_ATTR_PATH.gameHashes" \ diff --git a/pkgs/by-name/ne/nexusmods-app/vendored/README.md b/pkgs/by-name/ne/nexusmods-app/vendored/README.md deleted file mode 100644 index f92cbf87e634..000000000000 --- a/pkgs/by-name/ne/nexusmods-app/vendored/README.md +++ /dev/null @@ -1,31 +0,0 @@ -This directory contains a vendored copy of `games.json`, along with tooling to generate it. - -## Purpose - -The games data is fetched at runtime by NexusMods.App, however it is also included at build time for two reasons: - -1. It allows tests to run against real data. -2. It is used as cached data, speeding up the app's initial run. - -It is not vital for the file to contain all games, however ideally it should contain all games _supported_ by this version of NexusMods.App. -That way the initial run's cached data is more useful. - -If this file grows too large, because we are including too many games, we can patch the `csproj` build spec so that `games.json` is not used at build time. -We would also need to patch or disable any tests that rely on it. - -## Generating - -`games.json` is generated automatically by `update.sh`, using data from [nexusmods' API][url] and the games listed in `game-ids.nix`. - -To add a new game to `games.json`: -- Inspect the [nexusmods endpoint][url] to find the game's name and ID -- Add the name and ID to `game-ids.nix` -- Run `update.sh` -- Commit the result - -> [!Note] -> Running `update.sh` may also update the existing games, so you may wish to create two separate commits using `git add --patch`. -> One for updating the existing data and another for adding the new game. - -[url]: https://data.nexusmods.com/file/nexus-data/games.json - diff --git a/pkgs/by-name/ne/nexusmods-app/vendored/game-ids.nix b/pkgs/by-name/ne/nexusmods-app/vendored/game-ids.nix deleted file mode 100644 index 764658cd596d..000000000000 --- a/pkgs/by-name/ne/nexusmods-app/vendored/game-ids.nix +++ /dev/null @@ -1,12 +0,0 @@ -# This file lists games to be included in the vendored games.json file. -# It is not critical to include all games, other than those referenced by the test suite. -# Ideally, all games supported by the app will be included, as this can improve first-run performance. -{ - # keep-sorted start case=no numeric=yes - "Baldur's Gate 3" = 3474; - "Cyberpunk 2077" = 3333; - "Mount & Blade II: Bannerlord" = 3174; - "Skyrim Special Edition" = 1704; - "Stardew Valley" = 1303; - # keep-sorted end -} diff --git a/pkgs/by-name/ne/nexusmods-app/vendored/games.json b/pkgs/by-name/ne/nexusmods-app/vendored/games.json deleted file mode 100644 index 6d7b4767e012..000000000000 --- a/pkgs/by-name/ne/nexusmods-app/vendored/games.json +++ /dev/null @@ -1,72 +0,0 @@ -[ - { - "id": 1303, - "name": "Stardew Valley", - "name_lower": "stardew valley", - "forum_url": "https://forums.nexusmods.com/games/19-stardew-valley/", - "nexusmods_url": "https://www.nexusmods.com/stardewvalley", - "genre": "Simulation", - "file_count": 145440, - "downloads": 642501970, - "domain_name": "stardewvalley", - "approved_date": 1457432329, - "mods": 26417, - "collections": 1958 - }, - { - "id": 1704, - "name": "Skyrim Special Edition", - "name_lower": "skyrim special edition", - "forum_url": "https://forums.nexusmods.com/games/6-skyrim/", - "nexusmods_url": "https://www.nexusmods.com/skyrimspecialedition", - "genre": "RPG", - "file_count": 671705, - "downloads": 9497429062, - "domain_name": "skyrimspecialedition", - "approved_date": 1477480498, - "mods": 120722, - "collections": 4886 - }, - { - "id": 3174, - "name": "Mount & Blade II: Bannerlord", - "name_lower": "mount & blade ii: bannerlord", - "forum_url": "https://forums.nexusmods.com/games/9-mount-blade-ii-bannerlord/", - "nexusmods_url": "https://www.nexusmods.com/mountandblade2bannerlord", - "genre": "Strategy", - "file_count": 51310, - "downloads": 118459993, - "domain_name": "mountandblade2bannerlord", - "approved_date": 1582898627, - "mods": 6455, - "collections": 294 - }, - { - "id": 3333, - "name": "Cyberpunk 2077", - "name_lower": "cyberpunk 2077", - "forum_url": "https://forums.nexusmods.com/games/1-cyberpunk-2077/", - "nexusmods_url": "https://www.nexusmods.com/cyberpunk2077", - "genre": "Action", - "file_count": 127687, - "downloads": 981788054, - "domain_name": "cyberpunk2077", - "approved_date": 1607433331, - "mods": 18464, - "collections": 1613 - }, - { - "id": 3474, - "name": "Baldur's Gate 3", - "name_lower": "baldur's gate 3", - "forum_url": "https://forums.nexusmods.com/games/2-baldurs-gate-3/", - "nexusmods_url": "https://www.nexusmods.com/baldursgate3", - "genre": "RPG", - "file_count": 107598, - "downloads": 366648874, - "domain_name": "baldursgate3", - "approved_date": 1602863114, - "mods": 15345, - "collections": 1725 - } -] diff --git a/pkgs/by-name/ne/nexusmods-app/vendored/update.sh b/pkgs/by-name/ne/nexusmods-app/vendored/update.sh deleted file mode 100755 index ac44e25fae48..000000000000 --- a/pkgs/by-name/ne/nexusmods-app/vendored/update.sh +++ /dev/null @@ -1,53 +0,0 @@ -#!/usr/bin/env nix-shell -#! nix-shell -i bash -p bash curl jq - -set -eu -o pipefail - -url='https://data.nexusmods.com/file/nexus-data/games.json' -self=$(realpath "$0") -dir=$(dirname "$self") -tmp=$(mktemp) - -cd "$dir"/../../../../../ - -ids=$( - nix-instantiate --eval --json \ - --argstr file "$dir"/game-ids.nix \ - --expr '{file}: builtins.attrValues (import file)' -) - -echo "Fetching games data" >&2 -curl "$url" \ - --silent \ - --show-error \ - --location | - jq --argjson ids "$ids" \ - 'map(select( .id | IN($ids[]) )) | sort_by(.id)' \ - >"$tmp" - -echo "Validating result" >&2 -nix-instantiate --eval --strict \ - --argstr idsNix "$dir"/game-ids.nix \ - --argstr gamesJson "$tmp" \ - --expr ' - { - idsNix, - gamesJson, - lib ? import , - }: - let - ids = import idsNix; - games = lib.importJSON gamesJson; - in - lib.forEach games ( - { id, name, ... }: - lib.throwIfNot - (id == ids.${name}) - "${name}: id ${toString id} does not match ${toString ids.${name}}" - null - ) - ' \ - >/dev/null - -echo "Installing games.json to $dir" >&2 -mv --force "$tmp" "$dir"/games.json