Move all of NixOS to nixos/ in preparation of the repository merge
This commit is contained in:
Eelco Dolstra
220
nixos/maintainers/scripts/ec2/create-ebs-amis.py
Executable file
220
nixos/maintainers/scripts/ec2/create-ebs-amis.py
Executable file
@@ -0,0 +1,220 @@
|
||||
#! /usr/bin/env python
|
||||
|
||||
import os
|
||||
import sys
|
||||
import time
|
||||
import argparse
|
||||
import nixops.util
|
||||
from nixops import deployment
|
||||
from boto.ec2.blockdevicemapping import BlockDeviceMapping, BlockDeviceType
|
||||
import boto.ec2
|
||||
|
||||
parser = argparse.ArgumentParser(description='Create an EBS-backed NixOS AMI')
|
||||
parser.add_argument('--region', dest='region', required=True, help='EC2 region to create the image in')
|
||||
parser.add_argument('--keep', dest='keep', action='store_true', help='Keep NixOps machine after use')
|
||||
parser.add_argument('--hvm', dest='hvm', action='store_true', help='Create HVM image')
|
||||
parser.add_argument('--key', dest='key_name', action='store_true', help='Keypair used for HVM instance creation', default="rob")
|
||||
args = parser.parse_args()
|
||||
|
||||
instance_type = "cc1.4xlarge" if args.hvm else "m1.small"
|
||||
ebs_size = 8 if args.hvm else 20
|
||||
|
||||
|
||||
# Start a NixOS machine in the given region.
|
||||
f = open("ebs-creator-config.nix", "w")
|
||||
f.write('''{{
|
||||
resources.ec2KeyPairs.keypair.accessKeyId = "logicblox-dev";
|
||||
resources.ec2KeyPairs.keypair.region = "{0}";
|
||||
|
||||
machine =
|
||||
{{ pkgs, ... }}:
|
||||
{{
|
||||
deployment.ec2.accessKeyId = "logicblox-dev";
|
||||
deployment.ec2.region = "{0}";
|
||||
deployment.ec2.blockDeviceMapping."/dev/xvdg".size = pkgs.lib.mkOverride 10 {1};
|
||||
}};
|
||||
}}
|
||||
'''.format(args.region, ebs_size))
|
||||
f.close()
|
||||
|
||||
db = deployment.open_database(deployment.get_default_state_file())
|
||||
try:
|
||||
depl = deployment.open_deployment(db, "ebs-creator")
|
||||
except Exception:
|
||||
depl = deployment.create_deployment(db)
|
||||
depl.name = "ebs-creator"
|
||||
depl.auto_response = "y"
|
||||
depl.nix_exprs = [os.path.abspath("./ebs-creator.nix"), os.path.abspath("./ebs-creator-config.nix")]
|
||||
if not args.keep: depl.destroy_resources()
|
||||
depl.deploy(allow_reboot=True)
|
||||
|
||||
m = depl.machines['machine']
|
||||
|
||||
|
||||
# Do the installation.
|
||||
device="/dev/xvdg"
|
||||
if args.hvm:
|
||||
m.run_command('parted -s /dev/xvdg -- mklabel msdos')
|
||||
m.run_command('parted -s /dev/xvdg -- mkpart primary ext2 1M -1s')
|
||||
device="/dev/xvdg1"
|
||||
|
||||
m.run_command("if mountpoint -q /mnt; then umount /mnt; fi")
|
||||
m.run_command("mkfs.ext4 -L nixos {0}".format(device))
|
||||
m.run_command("mkdir -p /mnt")
|
||||
m.run_command("mount {0} /mnt".format(device))
|
||||
m.run_command("touch /mnt/.ebs")
|
||||
m.run_command("mkdir -p /mnt/etc/nixos")
|
||||
m.run_command("nix-channel --add http://nixos.org/channels/nixos-unstable")
|
||||
m.run_command("nix-channel --update")
|
||||
m.run_command("nixos-rebuild switch")
|
||||
version = m.run_command("nixos-version", capture_stdout=True).replace('"', '').rstrip()
|
||||
print >> sys.stderr, "NixOS version is {0}".format(version)
|
||||
m.upload_file("./amazon-base-config.nix", "/mnt/etc/nixos/configuration.nix")
|
||||
m.run_command("nixos-install")
|
||||
if args.hvm:
|
||||
m.run_command('cp /mnt/nix/store/*-grub-0.97*/lib/grub/i386-pc/* /mnt/boot/grub')
|
||||
m.run_command('sed -i "s|hd0|hd0,0|" /mnt/boot/grub/menu.lst')
|
||||
m.run_command('echo "(hd1) /dev/xvdg" > device.map')
|
||||
m.run_command('echo -e "root (hd1,0)\nsetup (hd1)" | grub --device-map=device.map --batch')
|
||||
|
||||
|
||||
m.run_command("umount /mnt")
|
||||
|
||||
|
||||
if args.hvm:
|
||||
ami_name = "nixos-{0}-x86_64-ebs-hvm".format(version)
|
||||
description = "NixOS {0} (x86_64; EBS root; hvm)".format(version)
|
||||
else:
|
||||
ami_name = "nixos-{0}-x86_64-ebs".format(version)
|
||||
description = "NixOS {0} (x86_64; EBS root)".format(version)
|
||||
|
||||
|
||||
# Wait for the snapshot to finish.
|
||||
def check():
|
||||
status = snapshot.update()
|
||||
print >> sys.stderr, "snapshot status is {0}".format(status)
|
||||
return status == '100%'
|
||||
|
||||
m.connect()
|
||||
volume = m._conn.get_all_volumes([], filters={'attachment.instance-id': m.resource_id, 'attachment.device': "/dev/sdg"})[0]
|
||||
if args.hvm:
|
||||
instance = m._conn.run_instances( image_id="ami-6a9e4503"
|
||||
, instance_type=instance_type
|
||||
, key_name=args.key_name
|
||||
, placement=m.zone
|
||||
, security_groups=["eelco-test"]).instances[0]
|
||||
nixops.util.check_wait(lambda: instance.update() == 'running', max_tries=120)
|
||||
instance.stop()
|
||||
nixops.util.check_wait(lambda: instance.update() == 'stopped', max_tries=120)
|
||||
old_root_volume = m._conn.get_all_volumes([], filters={'attachment.instance-id': instance.id, 'attachment.device': "/dev/sda1"})[0]
|
||||
old_root_volume.detach()
|
||||
volume.detach()
|
||||
nixops.util.check_wait(lambda: volume.update() == 'available', max_tries=120)
|
||||
nixops.util.check_wait(lambda: old_root_volume.update() == 'available', max_tries=120)
|
||||
volume.attach(instance.id, '/dev/sda1')
|
||||
nixops.util.check_wait(lambda: volume.update() == 'in-use', max_tries=120)
|
||||
|
||||
ami_id = m._conn.create_image(instance.id, ami_name, description)
|
||||
time.sleep(5)
|
||||
image = m._conn.get_all_images([ami_id])[0]
|
||||
nixops.util.check_wait(lambda: image.update() == 'available', max_tries=120)
|
||||
instance.terminate()
|
||||
|
||||
else:
|
||||
# Create a snapshot.
|
||||
snapshot = volume.create_snapshot(description=description)
|
||||
print >> sys.stderr, "created snapshot {0}".format(snapshot.id)
|
||||
|
||||
nixops.util.check_wait(check, max_tries=120)
|
||||
|
||||
m._conn.create_tags([snapshot.id], {'Name': ami_name})
|
||||
|
||||
if not args.keep: depl.destroy_resources()
|
||||
|
||||
# Register the image.
|
||||
aki = m._conn.get_all_images(filters={'manifest-location': '*pv-grub-hd0_1.03-x86_64*'})[0]
|
||||
print >> sys.stderr, "using kernel image {0} - {1}".format(aki.id, aki.location)
|
||||
|
||||
block_map = BlockDeviceMapping()
|
||||
block_map['/dev/sda'] = BlockDeviceType(snapshot_id=snapshot.id, delete_on_termination=True)
|
||||
block_map['/dev/sdb'] = BlockDeviceType(ephemeral_name="ephemeral0")
|
||||
block_map['/dev/sdc'] = BlockDeviceType(ephemeral_name="ephemeral1")
|
||||
block_map['/dev/sdd'] = BlockDeviceType(ephemeral_name="ephemeral2")
|
||||
block_map['/dev/sde'] = BlockDeviceType(ephemeral_name="ephemeral3")
|
||||
|
||||
ami_id = m._conn.register_image(
|
||||
name=ami_name,
|
||||
description=description,
|
||||
architecture="x86_64",
|
||||
root_device_name="/dev/sda",
|
||||
kernel_id=aki.id,
|
||||
block_device_map=block_map)
|
||||
|
||||
print >> sys.stderr, "registered AMI {0}".format(ami_id)
|
||||
|
||||
print >> sys.stderr, "sleeping a bit..."
|
||||
time.sleep(30)
|
||||
|
||||
print >> sys.stderr, "setting image name..."
|
||||
m._conn.create_tags([ami_id], {'Name': ami_name})
|
||||
|
||||
print >> sys.stderr, "making image public..."
|
||||
image = m._conn.get_all_images(image_ids=[ami_id])[0]
|
||||
image.set_launch_permissions(user_ids=[], group_names=["all"])
|
||||
|
||||
# Do a test deployment to make sure that the AMI works.
|
||||
f = open("ebs-test.nix", "w")
|
||||
f.write(
|
||||
'''
|
||||
{{
|
||||
network.description = "NixOS EBS test";
|
||||
|
||||
resources.ec2KeyPairs.keypair.accessKeyId = "logicblox-dev";
|
||||
resources.ec2KeyPairs.keypair.region = "{0}";
|
||||
|
||||
machine = {{ config, pkgs, resources, ... }}: {{
|
||||
deployment.targetEnv = "ec2";
|
||||
deployment.ec2.accessKeyId = "logicblox-dev";
|
||||
deployment.ec2.region = "{0}";
|
||||
deployment.ec2.instanceType = "{2}";
|
||||
deployment.ec2.keyPair = resources.ec2KeyPairs.keypair.name;
|
||||
deployment.ec2.securityGroups = [ "admin" ];
|
||||
deployment.ec2.ami = "{1}";
|
||||
}};
|
||||
}}
|
||||
'''.format(args.region, ami_id, instance_type))
|
||||
f.close()
|
||||
|
||||
test_depl = deployment.create_deployment(db)
|
||||
test_depl.auto_response = "y"
|
||||
test_depl.name = "ebs-creator-test"
|
||||
test_depl.nix_exprs = [os.path.abspath("./ebs-test.nix")]
|
||||
test_depl.deploy(create_only=True)
|
||||
test_depl.machines['machine'].run_command("nixos-version")
|
||||
|
||||
if args.hvm:
|
||||
image_type = 'hvm'
|
||||
else:
|
||||
image_type = 'ebs'
|
||||
|
||||
# Log the AMI ID.
|
||||
f = open("{0}.{1}.ami-id".format(args.region, image_type), "w")
|
||||
f.write("{0}".format(ami_id))
|
||||
f.close()
|
||||
|
||||
for dest in [ 'us-east-1', 'us-west-1', 'us-west-2', 'eu-west-1']:
|
||||
if args.region != dest:
|
||||
print >> sys.stderr, "copying image from region {0} to {1}".format(args.region, dest)
|
||||
conn = boto.ec2.connect_to_region(dest)
|
||||
copy_image = conn.copy_image(args.region, ami_id, ami_name, description=None, client_token=None)
|
||||
|
||||
# Log the AMI ID.
|
||||
f = open("{0}.{1}.ami-id".format(dest, image_type), "w")
|
||||
f.write("{0}".format(copy_image.image_id))
|
||||
f.close()
|
||||
|
||||
|
||||
if not args.keep:
|
||||
test_depl.destroy_resources()
|
||||
test_depl.delete()
|
||||
|
||||
49
nixos/maintainers/scripts/ec2/create-s3-amis.sh
Executable file
49
nixos/maintainers/scripts/ec2/create-s3-amis.sh
Executable file
@@ -0,0 +1,49 @@
|
||||
#! /bin/sh -e
|
||||
|
||||
nixos=$(nix-instantiate --find-file nixos)
|
||||
export NIXOS_CONFIG=$(dirname $(readlink -f $0))/amazon-base-config.nix
|
||||
|
||||
version=$(nix-instantiate --eval-only '<nixos>' -A config.system.nixosVersion | sed s/'"'//g)
|
||||
echo "NixOS version is $version"
|
||||
|
||||
buildAndUploadFor() {
|
||||
system="$1"
|
||||
arch="$2"
|
||||
|
||||
echo "building $system image..."
|
||||
nix-build '<nixos>' \
|
||||
-A config.system.build.amazonImage --argstr system "$system" -o ec2-ami
|
||||
|
||||
ec2-bundle-image -i ./ec2-ami/nixos.img --user "$AWS_ACCOUNT" --arch "$arch" \
|
||||
-c "$EC2_CERT" -k "$EC2_PRIVATE_KEY"
|
||||
|
||||
for region in eu-west-1 us-east-1 us-west-1 us-west-2; do
|
||||
echo "uploading $system image for $region..."
|
||||
|
||||
name=nixos-$version-$arch-s3
|
||||
bucket="$(echo $name-$region | tr '[A-Z]_' '[a-z]-')"
|
||||
|
||||
if [ "$region" = eu-west-1 ]; then s3location=EU;
|
||||
elif [ "$region" = us-east-1 ]; then s3location=US;
|
||||
else s3location="$region"
|
||||
fi
|
||||
|
||||
ec2-upload-bundle -b "$bucket" -m /tmp/nixos.img.manifest.xml \
|
||||
-a "$EC2_ACCESS_KEY" -s "$EC2_SECRET_KEY" --location "$s3location" \
|
||||
--url http://s3.amazonaws.com
|
||||
|
||||
kernel=$(ec2-describe-images -o amazon --filter "manifest-location=*pv-grub-hd0_1.03-$arch*" --region "$region" | cut -f 2)
|
||||
echo "using PV-GRUB kernel $kernel"
|
||||
|
||||
ami=$(ec2-register "$bucket/nixos.img.manifest.xml" -n "$name" -d "NixOS $system r$revision" \
|
||||
--region "$region" --kernel "$kernel" | cut -f 2)
|
||||
|
||||
echo "AMI ID is $ami"
|
||||
|
||||
echo $ami >> $region.s3.ami-id
|
||||
|
||||
ec2-modify-image-attribute --region "$region" "$ami" -l -a all
|
||||
done
|
||||
}
|
||||
|
||||
buildAndUploadFor x86_64-linux x86_64
|
||||
13
nixos/maintainers/scripts/ec2/ebs-creator.nix
Normal file
13
nixos/maintainers/scripts/ec2/ebs-creator.nix
Normal file
@@ -0,0 +1,13 @@
|
||||
{
|
||||
network.description = "NixOS EBS creator";
|
||||
|
||||
machine =
|
||||
{ config, pkgs, resources, ... }:
|
||||
{ deployment.targetEnv = "ec2";
|
||||
deployment.ec2.instanceType = "m1.large";
|
||||
deployment.ec2.securityGroups = [ "admin" ];
|
||||
deployment.ec2.ebsBoot = false;
|
||||
deployment.ec2.keyPair = resources.ec2KeyPairs.keypair.name;
|
||||
environment.systemPackages = [ pkgs.parted ];
|
||||
};
|
||||
}
|
||||
Reference in New Issue
Block a user