androidenv: fix autoupdate and custom XMLs
This regression happened during refactoring of mkrepo.rb. Do another round of it with a focus on fixing the autoupdate and use of custom repository XMLs. Add an example to check this behavior (but don't run it in the test suite, since it requires IFD).
This commit is contained in:
Morgan Jones
@@ -1,15 +1,13 @@
|
||||
# How to update
|
||||
|
||||
1. `./fetchrepo.sh`
|
||||
2. `./mkrepo.sh`
|
||||
3. Check the `repo.json` diff for new stable versions of `tools`, `platform-tools`, `build-tools`, `emulator` and/or `ndk`
|
||||
4. Update the relevant argument defaults in `compose-android-packages.nix`
|
||||
`nix-shell maintainers/scripts/update.nix --argstr package androidenv.test-suite --argstr commit true`
|
||||
|
||||
# How to run tests
|
||||
|
||||
You may need to make yourself familiar with [package tests](../../../README.md#package-tests), and [Writing larger package tests](../../../README.md#writing-larger-package-tests), then run tests locally with:
|
||||
|
||||
```shell
|
||||
$ export NIXPKGS_ALLOW_UNFREE=1
|
||||
$ cd path/to/nixpkgs
|
||||
$ nix-build -A androidenv.test-suite.tests
|
||||
$ nix-build -A androidenv.test-suite
|
||||
```
|
||||
|
||||
@@ -31,7 +31,7 @@ in
|
||||
# Reads the repo JSON. If repoXmls is provided, will build a repo JSON into the Nix store.
|
||||
if repoXmls != null then
|
||||
let
|
||||
# Uses mkrepo.rb to create a repo spec.
|
||||
# Uses update.rb to create a repo spec.
|
||||
mkRepoJson =
|
||||
{
|
||||
packages ? [ ],
|
||||
@@ -43,6 +43,7 @@ in
|
||||
ruby.withPackages (
|
||||
pkgs: with pkgs; [
|
||||
slop
|
||||
curb
|
||||
nokogiri
|
||||
]
|
||||
)
|
||||
@@ -68,7 +69,7 @@ in
|
||||
preferLocalBuild = true;
|
||||
unpackPhase = "true";
|
||||
buildPhase = ''
|
||||
ruby ${./mkrepo.rb} ${lib.escapeShellArgs mkRepoRubyArguments} > repo.json
|
||||
env ruby -e 'load "${./update.rb}"' -- ${lib.escapeShellArgs mkRepoRubyArguments} --input /dev/null --output repo.json
|
||||
'';
|
||||
installPhase = ''
|
||||
mv repo.json $out
|
||||
|
||||
@@ -0,0 +1,60 @@
|
||||
{
|
||||
# If you want to use the in-tree version of nixpkgs:
|
||||
pkgs ? import ../../../../.. {
|
||||
config.allowUnfree = true;
|
||||
},
|
||||
|
||||
licenseAccepted ? pkgs.callPackage ../license.nix { },
|
||||
}:
|
||||
|
||||
# Tests IFD with androidenv. Needs a folder of `../xml` in your local tree;
|
||||
# use ../fetchrepo.sh to produce it.
|
||||
let
|
||||
androidEnv = pkgs.callPackage ./.. {
|
||||
inherit pkgs licenseAccepted;
|
||||
};
|
||||
|
||||
sdkArgs = {
|
||||
repoXmls = {
|
||||
packages = [ ../xml/repository2-3.xml ];
|
||||
images = [
|
||||
../xml/android-sys-img2-3.xml
|
||||
../xml/android-tv-sys-img2-3.xml
|
||||
../xml/google_apis-sys-img2-3.xml
|
||||
../xml/google_apis_playstore-sys-img2-3.xml
|
||||
../xml/android-wear-sys-img2-3.xml
|
||||
../xml/android-wear-cn-sys-img2-3.xml
|
||||
../xml/android-automotive-sys-img2-3.xml
|
||||
];
|
||||
addons = [ ../xml/addon2-3.xml ];
|
||||
};
|
||||
};
|
||||
|
||||
androidComposition = androidEnv.composeAndroidPackages sdkArgs;
|
||||
androidSdk = androidComposition.androidsdk;
|
||||
platformTools = androidComposition.platform-tools;
|
||||
jdk = pkgs.jdk;
|
||||
in
|
||||
pkgs.mkShell {
|
||||
name = "androidenv-example-ifd-demo";
|
||||
packages = [
|
||||
androidSdk
|
||||
platformTools
|
||||
jdk
|
||||
];
|
||||
|
||||
LANG = "C.UTF-8";
|
||||
LC_ALL = "C.UTF-8";
|
||||
JAVA_HOME = jdk.home;
|
||||
|
||||
# Note: ANDROID_HOME is deprecated. Use ANDROID_SDK_ROOT.
|
||||
ANDROID_SDK_ROOT = "${androidSdk}/libexec/android-sdk";
|
||||
|
||||
shellHook = ''
|
||||
# Write out local.properties for Android Studio.
|
||||
cat <<EOF > local.properties
|
||||
# This file was automatically generated by nix-shell.
|
||||
sdk.dir=$ANDROID_SDK_ROOT
|
||||
EOF
|
||||
'';
|
||||
}
|
||||
@@ -1,19 +0,0 @@
|
||||
#!/usr/bin/env nix-shell
|
||||
#!nix-shell -i bash -p "ruby.withPackages (pkgs: with pkgs; [ slop nokogiri moreutils ])"
|
||||
|
||||
set -e
|
||||
|
||||
pushd "$(dirname "$0")" &>/dev/null || exit 1
|
||||
|
||||
echo "Writing repo.json" >&2
|
||||
ruby mkrepo.rb \
|
||||
--packages ./xml/repository2-3.xml \
|
||||
--images ./xml/android-sys-img2-3.xml \
|
||||
--images ./xml/android-tv-sys-img2-3.xml \
|
||||
--images ./xml/android-wear-cn-sys-img2-3.xml \
|
||||
--images ./xml/android-wear-sys-img2-3.xml \
|
||||
--images ./xml/android-automotive-sys-img2-3.xml \
|
||||
--images ./xml/google_apis-sys-img2-3.xml \
|
||||
--images ./xml/google_apis_playstore-sys-img2-3.xml \
|
||||
--addons ./xml/addon2-3.xml <./repo.json
|
||||
popd &>/dev/null
|
||||
@@ -18,6 +18,7 @@ let
|
||||
in
|
||||
stdenv.mkDerivation {
|
||||
name = "androidenv-test-suite";
|
||||
version = "1";
|
||||
buildInputs = lib.mapAttrsToList (name: value: value) all-tests;
|
||||
|
||||
buildCommand = ''
|
||||
@@ -26,9 +27,8 @@ stdenv.mkDerivation {
|
||||
|
||||
passthru.tests = all-tests;
|
||||
|
||||
# This is the toplevel package, so inherit the update script
|
||||
passthru.updateScript = {
|
||||
command = [ ./update.sh ];
|
||||
command = [ ./update.rb ];
|
||||
attrPath = "androidenv.test-suite";
|
||||
supportedFeatures = [ "commit" ];
|
||||
};
|
||||
|
||||
Regular → Executable
+100
-53
@@ -1,11 +1,15 @@
|
||||
#!/usr/bin/env ruby
|
||||
#!/usr/bin/env nix-shell
|
||||
#!nix-shell -i ruby -p "ruby.withPackages (ps: with ps; [ slop curb nokogiri ])"
|
||||
|
||||
require 'json'
|
||||
require 'rubygems'
|
||||
require 'nokogiri'
|
||||
require 'slop'
|
||||
require 'shellwords'
|
||||
require 'erb'
|
||||
require 'uri'
|
||||
require 'stringio'
|
||||
require 'slop'
|
||||
require 'curb'
|
||||
require 'nokogiri'
|
||||
|
||||
# Returns a repo URL for a given package name.
|
||||
def repo_url value
|
||||
@@ -32,6 +36,36 @@ def image_url value, dir
|
||||
end
|
||||
end
|
||||
|
||||
# Runs a GET with curl.
|
||||
def _curl_get url
|
||||
curl = Curl::Easy.new(url) do |http|
|
||||
http.headers['User-Agent'] = 'nixpkgs androidenv update bot'
|
||||
yield http if block_given?
|
||||
end
|
||||
STDERR.print "GET #{url}"
|
||||
curl.perform
|
||||
STDERR.puts "... #{curl.response_code}"
|
||||
|
||||
StringIO.new(curl.body_str)
|
||||
end
|
||||
|
||||
# Retrieves a repo from the filesystem or a URL.
|
||||
def get location
|
||||
uri = URI.parse(location)
|
||||
case uri.scheme
|
||||
when 'repo'
|
||||
_curl_get repo_url("#{uri.host}#{uri.fragment}.xml")
|
||||
when 'image'
|
||||
_curl_get image_url("sys-img#{uri.fragment}.xml", uri.host)
|
||||
else
|
||||
if File.exist?(uri.path)
|
||||
File.open(uri.path, 'rt')
|
||||
else
|
||||
raise "Repository #{uri} was neither a file nor a repo URL"
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
# Returns a JSON with the data and structure of the input XML
|
||||
def to_json_collector doc
|
||||
json = {}
|
||||
@@ -438,9 +472,19 @@ def merge dest, src
|
||||
end
|
||||
|
||||
opts = Slop.parse do |o|
|
||||
o.array '-p', '--packages', 'packages repo XMLs to parse'
|
||||
o.array '-i', '--images', 'system image repo XMLs to parse'
|
||||
o.array '-a', '--addons', 'addon repo XMLs to parse'
|
||||
o.array '-p', '--packages', 'packages repo XMLs to parse', default: %w[repo://repository#2-3]
|
||||
o.array '-i', '--images', 'system image repo XMLs to parse', default: %w[
|
||||
image://android#2-3
|
||||
image://android-tv#2-3
|
||||
image://android-wear#2-3
|
||||
image://android-wear-cn#2-3
|
||||
image://android-automotive#2-3
|
||||
image://google_apis#2-3
|
||||
image://google_apis_playstore#2-3
|
||||
]
|
||||
o.array '-a', '--addons', 'addon repo XMLs to parse', default: %w[repo://addon#2-3]
|
||||
o.string '-I', '--input', 'input JSON file for repo', default: File.join(__dir__, 'repo.json')
|
||||
o.string '-O', '--output', 'output JSON file for repo', default: File.join(__dir__, 'repo.json')
|
||||
end
|
||||
|
||||
result = {}
|
||||
@@ -451,20 +495,20 @@ result['addons'] = {}
|
||||
result['extras'] = {}
|
||||
|
||||
opts[:packages].each do |filename|
|
||||
licenses, packages, extras = parse_package_xml(Nokogiri::XML(File.open(filename)) { |conf| conf.noblanks })
|
||||
licenses, packages, extras = parse_package_xml(Nokogiri::XML(get(filename)) { |conf| conf.noblanks })
|
||||
merge result['licenses'], licenses
|
||||
merge result['packages'], packages
|
||||
merge result['extras'], extras
|
||||
end
|
||||
|
||||
opts[:images].each do |filename|
|
||||
licenses, images = parse_image_xml(Nokogiri::XML(File.open(filename)) { |conf| conf.noblanks })
|
||||
licenses, images = parse_image_xml(Nokogiri::XML(get(filename)) { |conf| conf.noblanks })
|
||||
merge result['licenses'], licenses
|
||||
merge result['images'], images
|
||||
end
|
||||
|
||||
opts[:addons].each do |filename|
|
||||
licenses, addons, extras = parse_addon_xml(Nokogiri::XML(File.open(filename)) { |conf| conf.noblanks })
|
||||
licenses, addons, extras = parse_addon_xml(Nokogiri::XML(get(filename)) { |conf| conf.noblanks })
|
||||
merge result['licenses'], licenses
|
||||
merge result['addons'], addons
|
||||
merge result['extras'], extras
|
||||
@@ -491,7 +535,14 @@ two_years_ago = today - 2 * 365
|
||||
input = {}
|
||||
prev_latest = {}
|
||||
begin
|
||||
input_json = (STDIN.tty?) ? "{}" : STDIN.read
|
||||
input_json = if File.exist?(opts[:input])
|
||||
STDERR.puts "Reading #{opts[:input]}"
|
||||
File.read(opts[:input])
|
||||
else
|
||||
STDERR.puts "Creating new repo"
|
||||
"{}"
|
||||
end
|
||||
|
||||
if input_json != nil && !input_json.empty?
|
||||
input = expire_records(JSON.parse(input_json), two_years_ago)
|
||||
|
||||
@@ -511,40 +562,37 @@ fixup_result = fixup(result)
|
||||
# therefore the old packages will work as long as the links are working on the Google servers.
|
||||
output = merge input, fixup_result
|
||||
|
||||
# See if there are any changes in the latest versions.
|
||||
cur_latest = output['latest'] || {}
|
||||
|
||||
old_versions = []
|
||||
new_versions = []
|
||||
changes = []
|
||||
changed = false
|
||||
|
||||
cur_latest.each do |k, v|
|
||||
prev = prev_latest[k]
|
||||
if prev && prev != v
|
||||
old_versions << "#{k}:#{prev}"
|
||||
new_versions << "#{k}:#{v}"
|
||||
changes << "#{k}: #{prev} -> #{v}"
|
||||
changed = true
|
||||
end
|
||||
end
|
||||
|
||||
# Write the repository. Append a \n to keep nixpkgs Github Actions happy.
|
||||
File.write 'repo.json', (JSON.pretty_generate(sort_recursively(output)) + "\n")
|
||||
STDERR.puts "Writing #{opts[:output]}"
|
||||
File.write opts[:output], (JSON.pretty_generate(sort_recursively(output)) + "\n")
|
||||
|
||||
# Output metadata for the nixpkgs update script.
|
||||
changed_paths = []
|
||||
if changed
|
||||
if ENV['UPDATE_NIX_ATTR_PATH']
|
||||
# Instantiate it.
|
||||
test_result = `NIXPKGS_ALLOW_UNFREE=1 NIXPKGS_ACCEPT_ANDROID_SDK_LICENSE=1 nix-build ../../../../default.nix -A #{Shellwords.join [ENV['UPDATE_NIX_ATTR_PATH']]} 2>&1`
|
||||
test_status = $?.exitstatus
|
||||
tests_ran = true
|
||||
else
|
||||
tests_ran = false
|
||||
if ENV['UPDATE_NIX_ATTR_PATH']
|
||||
# See if there are any changes in the latest versions.
|
||||
cur_latest = output['latest'] || {}
|
||||
|
||||
old_versions = []
|
||||
new_versions = []
|
||||
changes = []
|
||||
changed = false
|
||||
|
||||
cur_latest.each do |k, v|
|
||||
prev = prev_latest[k]
|
||||
if prev && prev != v
|
||||
old_versions << "#{k}:#{prev}"
|
||||
new_versions << "#{k}:#{v}"
|
||||
changes << "#{k}: #{prev} -> #{v}"
|
||||
changed = true
|
||||
end
|
||||
end
|
||||
|
||||
template = ERB.new(<<-EOF, trim_mode: '<>-')
|
||||
changed_paths = []
|
||||
if changed
|
||||
# Instantiate it.
|
||||
test_result = `NIXPKGS_ALLOW_UNFREE=1 NIXPKGS_ACCEPT_ANDROID_SDK_LICENSE=1 nix-build #{Shellwords.escape(File.realpath(File.join(__dir__, '..', '..', '..', '..', 'default.nix')))} -A #{Shellwords.join [ENV['UPDATE_NIX_ATTR_PATH']]} 2>&1`
|
||||
test_status = $?.exitstatus
|
||||
|
||||
template = ERB.new(<<-EOF, trim_mode: '<>-')
|
||||
androidenv: <%= changes.join('; ') %>
|
||||
|
||||
Performed the following automatic androidenv updates:
|
||||
@@ -553,7 +601,6 @@ Performed the following automatic androidenv updates:
|
||||
- <%= change -%>
|
||||
<% end %>
|
||||
|
||||
<% if tests_ran %>
|
||||
Tests exited with status: <%= test_status -%>
|
||||
|
||||
<% if !test_result.empty? %>
|
||||
@@ -562,19 +609,19 @@ Last 100 lines of output:
|
||||
<%= test_result.lines.last(100).join -%>
|
||||
```
|
||||
<% end %>
|
||||
<% end %>
|
||||
EOF
|
||||
|
||||
changed_paths << {
|
||||
attrPath: 'androidenv.androidPkgs.androidsdk',
|
||||
oldVersion: old_versions.join('; '),
|
||||
newVersion: new_versions.join('; '),
|
||||
files: [
|
||||
File.realpath('repo.json')
|
||||
],
|
||||
commitMessage: template.result(binding)
|
||||
}
|
||||
end
|
||||
changed_paths << {
|
||||
attrPath: 'androidenv.androidPkgs.androidsdk',
|
||||
oldVersion: old_versions.join('; '),
|
||||
newVersion: new_versions.join('; '),
|
||||
files: [
|
||||
opts[:output]
|
||||
],
|
||||
commitMessage: template.result(binding)
|
||||
}
|
||||
end
|
||||
|
||||
# nix-update info is on stderr
|
||||
STDOUT.puts JSON.pretty_generate(changed_paths)
|
||||
# nix-update info is on stdout
|
||||
STDOUT.puts JSON.pretty_generate(changed_paths)
|
||||
end
|
||||
@@ -1,5 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
pushd "$(dirname "$0")" &>/dev/null || exit 1
|
||||
./fetchrepo.sh && ./mkrepo.sh
|
||||
popd &>/dev/null
|
||||
Reference in New Issue
Block a user