From 5ce8c3404395d5b6ef92eb6c456eaeeea1964c67 Mon Sep 17 00:00:00 2001 From: Ryan Horiguchi Date: Sat, 23 Apr 2022 01:13:02 +0200 Subject: [PATCH 1/9] gnomeExtensions: update-extensions.py format --- .../gnome/extensions/update-extensions.py | 26 +++++++++---------- 1 file changed, 12 insertions(+), 14 deletions(-) diff --git a/pkgs/desktops/gnome/extensions/update-extensions.py b/pkgs/desktops/gnome/extensions/update-extensions.py index 581781fb11a8..84eed2093024 100755 --- a/pkgs/desktops/gnome/extensions/update-extensions.py +++ b/pkgs/desktops/gnome/extensions/update-extensions.py @@ -1,16 +1,16 @@ #!/usr/bin/env nix-shell #!nix-shell -I nixpkgs=../../../.. -i python3 -p python3 -import json -import urllib.request -import urllib.error -from typing import List, Dict, Optional, Any, Tuple -import logging -from operator import itemgetter -import subprocess -import zipfile -import io import base64 +import io +import json +import logging +import subprocess +import urllib.error +import urllib.request +import zipfile +from operator import itemgetter +from typing import List, Dict, Optional, Any, Tuple # We don't want all those deprecated legacy extensions # Group extensions by GNOME "major" version for compatibility reasons @@ -21,14 +21,12 @@ supported_versions = { "42": "42", } - # Some type alias to increase readility of complex compound types PackageName = str ShellVersion = str Uuid = str ExtensionVersion = int - # Keep track of all names that have been used till now to detect collisions. # This works because we deterministically process all extensions in historical order # The outer dict level is the shell version, as we are tracking duplicates only per same Shell version. @@ -69,7 +67,7 @@ def fetch_extension_data(uuid: str, version: str) -> Tuple[str, str]: def generate_extension_versions( - extension_version_map: Dict[ShellVersion, ExtensionVersion], uuid: str + extension_version_map: Dict[ShellVersion, ExtensionVersion], uuid: str ) -> Dict[ShellVersion, Dict[str, str]]: """ Takes in a mapping from shell versions to extension versions and transforms it the way we need it: @@ -127,7 +125,7 @@ def pname_from_url(url: str) -> Tuple[str, str]: """ url = url.split("/") # type: ignore - return (url[3], url[2]) + return url[3], url[2] def process_extension(extension: Dict[str, Any]) -> Optional[Dict[str, Any]]: @@ -225,7 +223,7 @@ def scrape_extensions_index() -> List[Dict[str, Any]]: logging.info("Scraping page " + str(page)) try: with urllib.request.urlopen( - f"https://extensions.gnome.org/extension-query/?n_per_page=25&page={page}" + f"https://extensions.gnome.org/extension-query/?n_per_page=25&page={page}" ) as response: data = json.loads(response.read().decode())["extensions"] responseLength = len(data) From bce4a670cc2ffbbbf222249bb2442a73509b68ae Mon Sep 17 00:00:00 2001 From: Ryan Horiguchi Date: Sat, 23 Apr 2022 01:16:40 +0200 Subject: [PATCH 2/9] gnomeExtensions: update-extensions.py fix where metadata.json missing --- pkgs/desktops/gnome/extensions/update-extensions.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/pkgs/desktops/gnome/extensions/update-extensions.py b/pkgs/desktops/gnome/extensions/update-extensions.py index 84eed2093024..619569f8d3d7 100755 --- a/pkgs/desktops/gnome/extensions/update-extensions.py +++ b/pkgs/desktops/gnome/extensions/update-extensions.py @@ -46,6 +46,10 @@ def fetch_extension_data(uuid: str, version: str) -> Tuple[str, str]: uuid = uuid.replace("@", "") url: str = f"https://extensions.gnome.org/extension-data/{uuid}.v{version}.shell-extension.zip" + # TODO remove when Vitals@CoreCoding.com version != 53, this extension has a missing manifest.json + if url == 'https://extensions.gnome.org/extension-data/VitalsCoreCoding.com.v53.shell-extension.zip': + url = 'https://extensions.gnome.org/extension-data/VitalsCoreCoding.com.v53.shell-extension_v1BI2FB.zip' + # Yes, we download that file three times: # The first time is for the maintainter, so they may have a personal backup to fix potential issues From 30818451447ab1c076caa5c36fc0310895f2ab3d Mon Sep 17 00:00:00 2001 From: Ryan Horiguchi Date: Sat, 23 Apr 2022 01:19:38 +0200 Subject: [PATCH 3/9] gnomeExtensions: update-extensions.py fix typos --- pkgs/desktops/gnome/extensions/update-extensions.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/pkgs/desktops/gnome/extensions/update-extensions.py b/pkgs/desktops/gnome/extensions/update-extensions.py index 619569f8d3d7..3430e46d0401 100755 --- a/pkgs/desktops/gnome/extensions/update-extensions.py +++ b/pkgs/desktops/gnome/extensions/update-extensions.py @@ -21,7 +21,7 @@ supported_versions = { "42": "42", } -# Some type alias to increase readility of complex compound types +# Some type alias to increase readability of complex compound types PackageName = str ShellVersion = str Uuid = str @@ -52,7 +52,7 @@ def fetch_extension_data(uuid: str, version: str) -> Tuple[str, str]: # Yes, we download that file three times: - # The first time is for the maintainter, so they may have a personal backup to fix potential issues + # The first time is for the maintainer, so they may have a personal backup to fix potential issues # subprocess.run( # ["wget", url], capture_output=True, text=True # ) @@ -116,7 +116,7 @@ def generate_extension_versions( "version": str(extension_version), "sha256": sha256, # The downloads are impure, their metadata.json may change at any time. - # Thus, be back it up / pin it to remain deterministic + # Thus, we back it up / pin it to remain deterministic # Upstream issue: https://gitlab.gnome.org/Infrastructure/extensions-web/-/issues/137 "metadata": metadata, } @@ -153,7 +153,7 @@ def process_extension(extension: Dict[str, Any]) -> Optional[Dict[str, Any]]: Don't make any assumptions on it, and treat it like an opaque string! "link" follows the following schema: "/extension/$number/$string/" The number is monotonically increasing and unique to every extension. - The string is usually derived from the extensions's name (but shortened, kebab-cased and URL friendly). + The string is usually derived from the extension name (but shortened, kebab-cased and URL friendly). It may diverge from the actual name. The keys of "shell_version_map" are GNOME Shell version numbers. From c88d12200cdfd049f2915922c7cc443ac13430b2 Mon Sep 17 00:00:00 2001 From: Ryan Horiguchi Date: Sat, 23 Apr 2022 01:20:37 +0200 Subject: [PATCH 4/9] gnomeExtensions: update-extensions.py improve collision warning --- pkgs/desktops/gnome/extensions/update-extensions.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkgs/desktops/gnome/extensions/update-extensions.py b/pkgs/desktops/gnome/extensions/update-extensions.py index 3430e46d0401..6a3637292f73 100755 --- a/pkgs/desktops/gnome/extensions/update-extensions.py +++ b/pkgs/desktops/gnome/extensions/update-extensions.py @@ -198,7 +198,7 @@ def process_extension(extension: Dict[str, Any]) -> Optional[Dict[str, Any]]: for shell_version in shell_version_map.keys(): if pname in package_name_registry[shell_version]: - logging.warning(f"Package name '{pname}' is colliding.") + logging.warning(f"Package name '{pname}' for GNOME '{shell_version}' is colliding.") package_name_registry[shell_version][pname].append(uuid) else: package_name_registry[shell_version][pname] = [uuid] From 51fdb8f239cb69f1e88bf38a5b125d83adff0f22 Mon Sep 17 00:00:00 2001 From: Ryan Horiguchi Date: Sat, 23 Apr 2022 01:23:46 +0200 Subject: [PATCH 5/9] gnomeExtensions: update-extensions.py use relative paths --- pkgs/desktops/gnome/extensions/update-extensions.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/pkgs/desktops/gnome/extensions/update-extensions.py b/pkgs/desktops/gnome/extensions/update-extensions.py index 6a3637292f73..c5542f03d8eb 100755 --- a/pkgs/desktops/gnome/extensions/update-extensions.py +++ b/pkgs/desktops/gnome/extensions/update-extensions.py @@ -10,6 +10,7 @@ import urllib.error import urllib.request import zipfile from operator import itemgetter +from pathlib import Path from typing import List, Dict, Optional, Any, Tuple # We don't want all those deprecated legacy extensions @@ -35,6 +36,8 @@ package_name_registry: Dict[ShellVersion, Dict[PackageName, List[Uuid]]] = {} for shell_version in supported_versions.keys(): package_name_registry[shell_version] = {} +updater_dir_path = Path(__file__).resolve().parent + def fetch_extension_data(uuid: str, version: str) -> Tuple[str, str]: """ @@ -271,7 +274,7 @@ if __name__ == "__main__": f"Done. Writing results to extensions.json ({len(processed_extensions)} extensions in total)" ) - with open("extensions.json", "w") as out: + with open(updater_dir_path / "extensions.json", "w") as out: # Manually pretty-print the outer level, but then do one compact line per extension # This allows for the diffs to be manageable (one line of change per extension) despite their quantity for index, extension in enumerate(processed_extensions): @@ -283,14 +286,14 @@ if __name__ == "__main__": out.write("\n") out.write("]\n") - with open("extensions.json", "r") as out: + with open(updater_dir_path / "extensions.json", "r") as out: # Check that the generated file actually is valid JSON, just to be sure json.load(out) logging.info( "Done. Writing name collisions to collisions.json (please check manually)" ) - with open("collisions.json", "w") as out: + with open(updater_dir_path / "collisions.json", "w") as out: # Filter out those that are not duplicates package_name_registry_filtered: Dict[ShellVersion, Dict[PackageName, List[Uuid]]] = { # The outer level keys are shell versions From dcb93d0080d17d9ef94afeaa17d484f6ca882508 Mon Sep 17 00:00:00 2001 From: Ryan Horiguchi Date: Sat, 23 Apr 2022 01:24:38 +0200 Subject: [PATCH 6/9] gnomeExtensions: update-extensions.py improve download --- .../gnome/extensions/update-extensions.py | 27 +++++++++++-------- 1 file changed, 16 insertions(+), 11 deletions(-) diff --git a/pkgs/desktops/gnome/extensions/update-extensions.py b/pkgs/desktops/gnome/extensions/update-extensions.py index c5542f03d8eb..7448864ffa9c 100755 --- a/pkgs/desktops/gnome/extensions/update-extensions.py +++ b/pkgs/desktops/gnome/extensions/update-extensions.py @@ -2,13 +2,11 @@ #!nix-shell -I nixpkgs=../../../.. -i python3 -p python3 import base64 -import io import json import logging import subprocess import urllib.error import urllib.request -import zipfile from operator import itemgetter from pathlib import Path from typing import List, Dict, Optional, Any, Tuple @@ -53,22 +51,29 @@ def fetch_extension_data(uuid: str, version: str) -> Tuple[str, str]: if url == 'https://extensions.gnome.org/extension-data/VitalsCoreCoding.com.v53.shell-extension.zip': url = 'https://extensions.gnome.org/extension-data/VitalsCoreCoding.com.v53.shell-extension_v1BI2FB.zip' - # Yes, we download that file three times: + # Yes, we download that file two times: # The first time is for the maintainer, so they may have a personal backup to fix potential issues # subprocess.run( # ["wget", url], capture_output=True, text=True # ) - # The second time, we extract the metadata.json because we need it too - with urllib.request.urlopen(url) as response: - data = zipfile.ZipFile(io.BytesIO(response.read()), 'r') - metadata = base64.b64encode(data.read('metadata.json')).decode() + # The second time, we add the file to store + process = subprocess.run( + ["nix-prefetch-url", "--unpack", "--print-path", url], capture_output=True, text=True + ) - # The third time is to get the file into the store and to get its hash - hash = subprocess.run( - ["nix-prefetch-url", "--unpack", url], capture_output=True, text=True - ).stdout.strip() + lines = process.stdout.splitlines() + + # Get hash from first line of nix-prefetch-url output + hash = lines[0].strip() + + # Get path from second line of nix-prefetch-url output + path = Path(lines[1].strip()) + + # Get metadata.json content from nix-store + with open(path / "metadata.json", "r") as out: + metadata = base64.b64encode(out.read().encode("ascii")).decode() return hash, metadata From 7a54239267392e0209137e12e2567cd5262d346d Mon Sep 17 00:00:00 2001 From: Ryan Horiguchi Date: Sat, 23 Apr 2022 01:25:09 +0200 Subject: [PATCH 7/9] gnomeExtensions: update-extensions.py move info after writing to file --- .../gnome/extensions/update-extensions.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/pkgs/desktops/gnome/extensions/update-extensions.py b/pkgs/desktops/gnome/extensions/update-extensions.py index 7448864ffa9c..849832a70066 100755 --- a/pkgs/desktops/gnome/extensions/update-extensions.py +++ b/pkgs/desktops/gnome/extensions/update-extensions.py @@ -275,10 +275,6 @@ if __name__ == "__main__": processed_extensions.append(processed_extension) logging.debug(f"Processed {num + 1} / {len(raw_extensions)}") - logging.info( - f"Done. Writing results to extensions.json ({len(processed_extensions)} extensions in total)" - ) - with open(updater_dir_path / "extensions.json", "w") as out: # Manually pretty-print the outer level, but then do one compact line per extension # This allows for the diffs to be manageable (one line of change per extension) despite their quantity @@ -291,13 +287,14 @@ if __name__ == "__main__": out.write("\n") out.write("]\n") + logging.info( + f"Done. Writing results to extensions.json ({len(processed_extensions)} extensions in total)" + ) + with open(updater_dir_path / "extensions.json", "r") as out: # Check that the generated file actually is valid JSON, just to be sure json.load(out) - logging.info( - "Done. Writing name collisions to collisions.json (please check manually)" - ) with open(updater_dir_path / "collisions.json", "w") as out: # Filter out those that are not duplicates package_name_registry_filtered: Dict[ShellVersion, Dict[PackageName, List[Uuid]]] = { @@ -309,3 +306,7 @@ if __name__ == "__main__": } json.dump(package_name_registry_filtered, out, indent=2, ensure_ascii=False) out.write("\n") + + logging.info( + "Done. Writing name collisions to collisions.json (please check manually)" + ) From d13f1f869f4d60a0b01cc7b7d3bbe8899a5d545b Mon Sep 17 00:00:00 2001 From: Ryan Horiguchi Date: Sat, 23 Apr 2022 01:34:52 +0200 Subject: [PATCH 8/9] gnomeExtensions: update-extensions.py make var more pythonic --- pkgs/desktops/gnome/extensions/update-extensions.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkgs/desktops/gnome/extensions/update-extensions.py b/pkgs/desktops/gnome/extensions/update-extensions.py index 849832a70066..f6e831c7b536 100755 --- a/pkgs/desktops/gnome/extensions/update-extensions.py +++ b/pkgs/desktops/gnome/extensions/update-extensions.py @@ -238,13 +238,13 @@ def scrape_extensions_index() -> List[Dict[str, Any]]: f"https://extensions.gnome.org/extension-query/?n_per_page=25&page={page}" ) as response: data = json.loads(response.read().decode())["extensions"] - responseLength = len(data) + response_length = len(data) for extension in data: extensions.append(extension) # If our page isn't "full", it must have been the last one - if responseLength < 25: + if response_length < 25: logging.debug( f"\tThis page only has {responseLength} entries, so it must be the last one." ) From d4268e79530b7e4b228216fefe233586edfcb575 Mon Sep 17 00:00:00 2001 From: Ryan Horiguchi Date: Sat, 23 Apr 2022 17:53:31 +0200 Subject: [PATCH 9/9] gnomeExtensions: update-extensions.py remove debug code --- pkgs/desktops/gnome/extensions/update-extensions.py | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/pkgs/desktops/gnome/extensions/update-extensions.py b/pkgs/desktops/gnome/extensions/update-extensions.py index f6e831c7b536..a2f42be06258 100755 --- a/pkgs/desktops/gnome/extensions/update-extensions.py +++ b/pkgs/desktops/gnome/extensions/update-extensions.py @@ -51,14 +51,7 @@ def fetch_extension_data(uuid: str, version: str) -> Tuple[str, str]: if url == 'https://extensions.gnome.org/extension-data/VitalsCoreCoding.com.v53.shell-extension.zip': url = 'https://extensions.gnome.org/extension-data/VitalsCoreCoding.com.v53.shell-extension_v1BI2FB.zip' - # Yes, we download that file two times: - - # The first time is for the maintainer, so they may have a personal backup to fix potential issues - # subprocess.run( - # ["wget", url], capture_output=True, text=True - # ) - - # The second time, we add the file to store + # Download extension and add the zip content to nix-store process = subprocess.run( ["nix-prefetch-url", "--unpack", "--print-path", url], capture_output=True, text=True )