Revert "nexusmods-app: fetch games.json from the Internet Archive"
This commit is contained in:
Matt Sturgeon
@@ -5,7 +5,6 @@
|
||||
desktop-file-utils,
|
||||
dotnetCorePackages,
|
||||
fetchFromGitHub,
|
||||
fetchurl,
|
||||
imagemagick,
|
||||
lib,
|
||||
xdg-utils,
|
||||
@@ -35,12 +34,6 @@ buildDotnetModule (finalAttrs: {
|
||||
|
||||
gameHashes = callPackage ./game-hashes { };
|
||||
|
||||
gamesJson = fetchurl {
|
||||
name = "games.json";
|
||||
url = "https://web.archive.org/web/20251221212501id_/https://data.nexusmods.com/file/nexus-data/games.json";
|
||||
hash = "sha256-OVy6b/n6n2/TdzfHCWDOTw2yai87ieki21fEp1IGamY=";
|
||||
};
|
||||
|
||||
enableParallelBuilding = false;
|
||||
|
||||
# If the whole solution is published, there seems to be a race condition where
|
||||
@@ -73,9 +66,9 @@ buildDotnetModule (finalAttrs: {
|
||||
substituteInPlace src/NexusMods.Games.FileHashes/NexusMods.Games.FileHashes.csproj \
|
||||
--replace-fail '$(BaseIntermediateOutputPath)games_hashes_db.zip' "$gameHashes"
|
||||
|
||||
# Use a pinned version of the nexus API's games.json data
|
||||
# Use a vendored version of the nexus API's games.json data
|
||||
substituteInPlace src/NexusMods.Networking.NexusWebApi/NexusMods.Networking.NexusWebApi.csproj \
|
||||
--replace-fail '$(BaseIntermediateOutputPath)games.json' "$gamesJson"
|
||||
--replace-fail '$(BaseIntermediateOutputPath)games.json' ${./vendored/games.json}
|
||||
|
||||
${lib.optionalString finalAttrs.doCheck ''
|
||||
# For some reason these tests fail (intermittently?) with a zero timestamp
|
||||
|
||||
@@ -1,78 +0,0 @@
|
||||
#!/usr/bin/env nix-shell
|
||||
#! nix-shell -i bash -p bash curl common-updater-scripts
|
||||
|
||||
set -eu -o pipefail
|
||||
|
||||
# Set a default attrpath to allow running this update script directly
|
||||
export UPDATE_NIX_ATTR_PATH="${UPDATE_NIX_ATTR_PATH:-"nexusmods-app"}"
|
||||
|
||||
save_url=https://web.archive.org/save
|
||||
json_url=https://data.nexusmods.com/file/nexus-data/games.json
|
||||
self=$(realpath "$0")
|
||||
dir=$(dirname "$self")
|
||||
curl_args=(
|
||||
"$save_url/$json_url"
|
||||
|
||||
--silent
|
||||
--fail
|
||||
--location # follow redirects
|
||||
--head # we only need headers
|
||||
--output /dev/null # discard body
|
||||
--write-out '%{url_effective}'
|
||||
|
||||
# Format --data-urlencode as GET query params
|
||||
--get
|
||||
# Instruct IA not to crawl linked content
|
||||
--data-urlencode capture_all=0
|
||||
# Only create a snapshot if older than 1 day
|
||||
--data-urlencode if_not_archived_within=86400
|
||||
)
|
||||
cd "$dir"/../../../../
|
||||
|
||||
# Ask the Internet Archive to save a snapshot of `games.json`
|
||||
# We capture its 'HTTP 302 Found' redirect location
|
||||
echo "Fetching games.json Internet Archive URL" >&2
|
||||
url=$(curl "${curl_args[@]}")
|
||||
|
||||
# Use raw replay mode to avoid rewritten links in the JSON content
|
||||
url=$(
|
||||
sed -E 's|^https://web\.archive\.org/web/([0-9]+)/|https://web.archive.org/web/\1id_/|' \
|
||||
<<<"$url"
|
||||
)
|
||||
|
||||
# The query parameters are only used in the snapshot request,
|
||||
# but remain in the redirected URL. Strip them:
|
||||
url="${url%%\?*}"
|
||||
|
||||
current_url=$(
|
||||
nix-instantiate --eval --raw \
|
||||
--attr "$UPDATE_NIX_ATTR_PATH.gamesJson.url"
|
||||
)
|
||||
if [ "$current_url" = "$url" ]; then
|
||||
echo "games.json has no changes" >&2
|
||||
exit
|
||||
fi
|
||||
|
||||
echo "Downloading and hashing games.json" >&2
|
||||
hash=$(
|
||||
nix --extra-experimental-features nix-command \
|
||||
hash convert --hash-algo sha256 --to sri \
|
||||
"$(nix-prefetch-url "$url" --type sha256)"
|
||||
)
|
||||
|
||||
# `update-source-version` needs the package version,
|
||||
# even though we're updating the games.json source
|
||||
package_version=$(
|
||||
nix-instantiate --eval --raw \
|
||||
--attr "$UPDATE_NIX_ATTR_PATH.version"
|
||||
)
|
||||
|
||||
echo "Updating games.json source" >&2
|
||||
update-source-version \
|
||||
"$UPDATE_NIX_ATTR_PATH" \
|
||||
"$package_version" \
|
||||
"$hash" \
|
||||
"$url" \
|
||||
--ignore-same-version \
|
||||
--source-key=gamesJson \
|
||||
--file="$dir"/package.nix
|
||||
@@ -10,28 +10,15 @@ self=$(realpath "$0")
|
||||
dir=$(dirname "$self")
|
||||
cd "$dir"/../../../../
|
||||
|
||||
current_version=$(
|
||||
nix-instantiate --eval --raw \
|
||||
--attr "$UPDATE_NIX_ATTR_PATH.version"
|
||||
)
|
||||
# Update vendored files
|
||||
"$dir"/vendored/update.sh
|
||||
|
||||
# Update game_hashes_db
|
||||
UPDATE_NIX_ATTR_PATH="$UPDATE_NIX_ATTR_PATH.gameHashes" \
|
||||
"$dir"/game-hashes/update.sh
|
||||
|
||||
url=$(
|
||||
nix-instantiate --eval --raw \
|
||||
--attr "$UPDATE_NIX_ATTR_PATH.meta.homepage"
|
||||
)
|
||||
|
||||
nix-update --url "$url"
|
||||
|
||||
new_version=$(
|
||||
nix-instantiate --eval --raw \
|
||||
--attr "$UPDATE_NIX_ATTR_PATH.version"
|
||||
)
|
||||
|
||||
if [ "$current_version" != "$new_version" ]; then
|
||||
# Update games.json
|
||||
"$dir"/update-games-json.sh
|
||||
|
||||
# Update game_hashes_db
|
||||
UPDATE_NIX_ATTR_PATH="$UPDATE_NIX_ATTR_PATH.gameHashes" \
|
||||
"$dir"/game-hashes/update.sh
|
||||
fi
|
||||
|
||||
@@ -0,0 +1,31 @@
|
||||
This directory contains a vendored copy of `games.json`, along with tooling to generate it.
|
||||
|
||||
## Purpose
|
||||
|
||||
The games data is fetched at runtime by NexusMods.App, however it is also included at build time for two reasons:
|
||||
|
||||
1. It allows tests to run against real data.
|
||||
2. It is used as cached data, speeding up the app's initial run.
|
||||
|
||||
It is not vital for the file to contain all games, however ideally it should contain all games _supported_ by this version of NexusMods.App.
|
||||
That way the initial run's cached data is more useful.
|
||||
|
||||
If this file grows too large, because we are including too many games, we can patch the `csproj` build spec so that `games.json` is not used at build time.
|
||||
We would also need to patch or disable any tests that rely on it.
|
||||
|
||||
## Generating
|
||||
|
||||
`games.json` is generated automatically by `update.sh`, using data from [nexusmods' API][url] and the games listed in `game-ids.nix`.
|
||||
|
||||
To add a new game to `games.json`:
|
||||
- Inspect the [nexusmods endpoint][url] to find the game's name and ID
|
||||
- Add the name and ID to `game-ids.nix`
|
||||
- Run `update.sh`
|
||||
- Commit the result
|
||||
|
||||
> [!Note]
|
||||
> Running `update.sh` may also update the existing games, so you may wish to create two separate commits using `git add --patch`.
|
||||
> One for updating the existing data and another for adding the new game.
|
||||
|
||||
[url]: https://data.nexusmods.com/file/nexus-data/games.json
|
||||
|
||||
@@ -0,0 +1,12 @@
|
||||
# This file lists games to be included in the vendored games.json file.
|
||||
# It is not critical to include all games, other than those referenced by the test suite.
|
||||
# Ideally, all games supported by the app will be included, as this can improve first-run performance.
|
||||
{
|
||||
# keep-sorted start case=no numeric=yes
|
||||
"Baldur's Gate 3" = 3474;
|
||||
"Cyberpunk 2077" = 3333;
|
||||
"Mount & Blade II: Bannerlord" = 3174;
|
||||
"Skyrim Special Edition" = 1704;
|
||||
"Stardew Valley" = 1303;
|
||||
# keep-sorted end
|
||||
}
|
||||
@@ -0,0 +1,72 @@
|
||||
[
|
||||
{
|
||||
"id": 1303,
|
||||
"name": "Stardew Valley",
|
||||
"name_lower": "stardew valley",
|
||||
"forum_url": "https://forums.nexusmods.com/games/19-stardew-valley/",
|
||||
"nexusmods_url": "https://www.nexusmods.com/stardewvalley",
|
||||
"genre": "Simulation",
|
||||
"file_count": 145440,
|
||||
"downloads": 642501970,
|
||||
"domain_name": "stardewvalley",
|
||||
"approved_date": 1457432329,
|
||||
"mods": 26417,
|
||||
"collections": 1958
|
||||
},
|
||||
{
|
||||
"id": 1704,
|
||||
"name": "Skyrim Special Edition",
|
||||
"name_lower": "skyrim special edition",
|
||||
"forum_url": "https://forums.nexusmods.com/games/6-skyrim/",
|
||||
"nexusmods_url": "https://www.nexusmods.com/skyrimspecialedition",
|
||||
"genre": "RPG",
|
||||
"file_count": 671705,
|
||||
"downloads": 9497429062,
|
||||
"domain_name": "skyrimspecialedition",
|
||||
"approved_date": 1477480498,
|
||||
"mods": 120722,
|
||||
"collections": 4886
|
||||
},
|
||||
{
|
||||
"id": 3174,
|
||||
"name": "Mount & Blade II: Bannerlord",
|
||||
"name_lower": "mount & blade ii: bannerlord",
|
||||
"forum_url": "https://forums.nexusmods.com/games/9-mount-blade-ii-bannerlord/",
|
||||
"nexusmods_url": "https://www.nexusmods.com/mountandblade2bannerlord",
|
||||
"genre": "Strategy",
|
||||
"file_count": 51310,
|
||||
"downloads": 118459993,
|
||||
"domain_name": "mountandblade2bannerlord",
|
||||
"approved_date": 1582898627,
|
||||
"mods": 6455,
|
||||
"collections": 294
|
||||
},
|
||||
{
|
||||
"id": 3333,
|
||||
"name": "Cyberpunk 2077",
|
||||
"name_lower": "cyberpunk 2077",
|
||||
"forum_url": "https://forums.nexusmods.com/games/1-cyberpunk-2077/",
|
||||
"nexusmods_url": "https://www.nexusmods.com/cyberpunk2077",
|
||||
"genre": "Action",
|
||||
"file_count": 127687,
|
||||
"downloads": 981788054,
|
||||
"domain_name": "cyberpunk2077",
|
||||
"approved_date": 1607433331,
|
||||
"mods": 18464,
|
||||
"collections": 1613
|
||||
},
|
||||
{
|
||||
"id": 3474,
|
||||
"name": "Baldur's Gate 3",
|
||||
"name_lower": "baldur's gate 3",
|
||||
"forum_url": "https://forums.nexusmods.com/games/2-baldurs-gate-3/",
|
||||
"nexusmods_url": "https://www.nexusmods.com/baldursgate3",
|
||||
"genre": "RPG",
|
||||
"file_count": 107598,
|
||||
"downloads": 366648874,
|
||||
"domain_name": "baldursgate3",
|
||||
"approved_date": 1602863114,
|
||||
"mods": 15345,
|
||||
"collections": 1725
|
||||
}
|
||||
]
|
||||
+53
@@ -0,0 +1,53 @@
|
||||
#!/usr/bin/env nix-shell
|
||||
#! nix-shell -i bash -p bash curl jq
|
||||
|
||||
set -eu -o pipefail
|
||||
|
||||
url='https://data.nexusmods.com/file/nexus-data/games.json'
|
||||
self=$(realpath "$0")
|
||||
dir=$(dirname "$self")
|
||||
tmp=$(mktemp)
|
||||
|
||||
cd "$dir"/../../../../../
|
||||
|
||||
ids=$(
|
||||
nix-instantiate --eval --json \
|
||||
--argstr file "$dir"/game-ids.nix \
|
||||
--expr '{file}: builtins.attrValues (import file)'
|
||||
)
|
||||
|
||||
echo "Fetching games data" >&2
|
||||
curl "$url" \
|
||||
--silent \
|
||||
--show-error \
|
||||
--location |
|
||||
jq --argjson ids "$ids" \
|
||||
'map(select( .id | IN($ids[]) )) | sort_by(.id)' \
|
||||
>"$tmp"
|
||||
|
||||
echo "Validating result" >&2
|
||||
nix-instantiate --eval --strict \
|
||||
--argstr idsNix "$dir"/game-ids.nix \
|
||||
--argstr gamesJson "$tmp" \
|
||||
--expr '
|
||||
{
|
||||
idsNix,
|
||||
gamesJson,
|
||||
lib ? import <nixpkgs/lib>,
|
||||
}:
|
||||
let
|
||||
ids = import idsNix;
|
||||
games = lib.importJSON gamesJson;
|
||||
in
|
||||
lib.forEach games (
|
||||
{ id, name, ... }:
|
||||
lib.throwIfNot
|
||||
(id == ids.${name})
|
||||
"${name}: id ${toString id} does not match ${toString ids.${name}}"
|
||||
null
|
||||
)
|
||||
' \
|
||||
>/dev/null
|
||||
|
||||
echo "Installing games.json to $dir" >&2
|
||||
mv --force "$tmp" "$dir"/games.json
|
||||
Reference in New Issue
Block a user