const aws = which("aws", { required: true });

let env;

if (isCI) {

env = {

AWS_ACCESS_KEY_ID: getSecret("EC2_ACCESS_KEY_ID", { required: true }),

AWS_SECRET_ACCESS_KEY: getSecret("EC2_SECRET_ACCESS_KEY", { required: true }),

AWS_REGION: getSecret("EC2_REGION", { required: false }) || "us-east-1",

};

}

const { stdout } = await spawnSafe($`${aws} ${args} --output json`, { env, ...options });

try {

return JSON.parse(stdout);

} catch {

return;

}

return Object.entries(options)

.filter(([_, value]) => typeof value !== "undefined")

.map(([key, value]) => `Name=${key},Values=${value}`);

return Object.entries(options)

.filter(([_, value]) => typeof value !== "undefined")

.map(([key, value]) => `--${key}=${value}`);

const filters = aws.getFilters(options);

const { Reservations } = await aws.spawn($`ec2 describe-instances --filters ${filters}`);

return Reservations.flatMap(({ Instances }) => Instances).sort((a, b) => (a.LaunchTime < b.LaunchTime ? 1 : -1));

for (let i = 0; i < 3; i++) {

const flags = aws.getFlags(options);

const result = await aws.spawn($`ec2 run-instances ${flags}`, {

throwOnError: error => {

if (options["instance-market-options"] && /InsufficientInstanceCapacity/i.test(inspect(error))) {

delete options["instance-market-options"];

const instanceType = options["instance-type"] || "default";

console.warn(`There is not enough capacity for ${instanceType} spot instances, retrying with on-demand...`);

return false;

}

return true;

},

});

if (result) {

const { Instances } = result;

if (Instances.length) {

return Instances.sort((a, b) => (a.LaunchTime < b.LaunchTime ? 1 : -1));

}

}

await new Promise(resolve => setTimeout(resolve, i * Math.random() * 15_000));

}

throw new Error(`Failed to run instances: ${inspect(instanceOptions)}`);

await aws.spawn($`ec2 terminate-instances --instance-ids ${instanceIds}`, {

throwOnError: error => !/InvalidInstanceID\.NotFound/i.test(inspect(error)),

});

await aws.spawn($`ec2 wait ${action} --instance-ids ${instanceIds}`, {

retryOnError: error => /max attempts exceeded/i.test(inspect(error)),

});

const attempts = passwordOptions.wait ? 15 : 1;

for (let i = 0; i < attempts; i++) {

const { PasswordData } = await aws.spawn($`ec2 get-password-data --instance-id ${instanceId}`);

if (PasswordData) {

return decryptPassword(PasswordData, privateKeyPath);

}

await new Promise(resolve => setTimeout(resolve, 60000 * i));

}

throw new Error(`Failed to get password data for instance: ${instanceId}`);

const { ["owner-alias"]: owners, ...filterOptions } = options;

const filters = aws.getFilters(filterOptions);

if (owners) {

filters.push(`--owners=${owners}`);

}

const { Images } = await aws.spawn($`ec2 describe-images --filters ${filters}`);

return Images.sort((a, b) => (a.CreationDate < b.CreationDate ? 1 : -1));

const flags = aws.getFlags(options);

/** @type {string | undefined} */

let existingImageId;

/** @type {AwsImage | undefined} */

const image = await aws.spawn($`ec2 create-image ${flags}`, {

throwOnError: error => {

const match = /already in use by AMI (ami-[a-z0-9]+)/i.exec(inspect(error));

if (!match) {

return true;

}

const [, imageId] = match;

existingImageId = imageId;

return false;

},

});

if (!existingImageId) {

const { ImageId } = image;

return ImageId;

}

await aws.spawn($`ec2 deregister-image --image-id ${existingImageId}`);

const { ImageId } = await aws.spawn($`ec2 create-image ${flags}`);

return ImageId;

const flags = aws.getFlags(options);

const { ImageId } = await aws.spawn($`ec2 copy-image ${flags}`);

return ImageId;

await aws.spawn($`ec2 wait ${action} --image-ids ${imageIds}`, {

retryOnError: error => /max attempts exceeded/i.test(inspect(error)),

});

const command = names

? $`ec2 describe-key-pairs --include-public-key --key-names ${names}`

: $`ec2 describe-key-pairs --include-public-key`;

const { KeyPairs } = await aws.spawn(command);

return KeyPairs;

const keyName = name || `key-pair-${sha256(publicKey)}`;

const publicKeyBase64 = Buffer.from(publicKey).toString("base64");

/** @type {AwsKeyPair | undefined} */

const keyPair = await aws.spawn(

$`ec2 import-key-pair --key-name ${keyName} --public-key-material ${publicKeyBase64}`,

{

throwOnError: error => !/InvalidKeyPair\.Duplicate/i.test(inspect(error)),

},

);

if (keyPair) {

return keyPair;

}

const keyPairs = await aws.describeKeyPairs(keyName);

if (keyPairs.length) {

return keyPairs[0];

}

throw new Error(`Failed to import key pair: ${keyName}`);

let imageId = imageOrImageId;

if (typeof imageOrImageId === "object") {

const { ImageId, State } = imageOrImageId;

if (State === "available") {

return imageOrImageId;

}

imageId = ImageId;

}

await aws.waitImage("image-available", imageId);

const [availableImage] = await aws.describeImages({

"state": "available",

"image-id": imageId,

});

if (!availableImage) {

throw new Error(`Failed to find available image: ${imageId}`);

}

return availableImage;

const { os, arch, distro, release } = options;

let name, owner;

if (os === "linux") {

if (!distro || distro === "debian") {

owner = "amazon";

name = `debian-${release || "*"}-${arch === "aarch64" ? "arm64" : "amd64"}-*`;

} else if (distro === "ubuntu") {

owner = "099720109477";

name = `ubuntu/images/hvm-ssd*/ubuntu-*-${release || "*"}-${arch === "aarch64" ? "arm64" : "amd64"}-server-*`;

} else if (distro === "amazonlinux") {

owner = "amazon";

if (release === "1" && arch === "x64") {

name = `amzn-ami-2018.03.*`;

} else if (release === "2") {

name = `amzn2-ami-hvm-*-${arch === "aarch64" ? "arm64" : "x86_64"}-gp2`;

} else {

name = `al${release || "*"}-ami-*-${arch === "aarch64" ? "arm64" : "x86_64"}`;

}

} else if (distro === "alpine") {

owner = "538276064493";

name = `alpine-${release || "*"}.*-${arch === "aarch64" ? "aarch64" : "x86_64"}-uefi-cloudinit-*`;

} else if (distro === "centos") {

owner = "aws-marketplace";

name = `CentOS-Stream-ec2-${release || "*"}-*.${arch === "aarch64" ? "aarch64" : "x86_64"}-*`;

}

} else if (os === "windows") {

if (!distro || distro === "server") {

owner = "amazon";

name = `Windows_Server-${release || "*"}-English-Full-Base-*`;

}

}

if (!name) {

throw new Error(`Unsupported platform: ${inspect(options)}`);

}

const baseImages = await aws.describeImages({

"state": "available",

"owner-alias": owner,

"name": name,

});

// console.table(baseImages.map(v => v.Name));

if (!baseImages.length) {

throw new Error(`No base image found: ${inspect(options)}`);

}

const [baseImage] = baseImages;

return aws.getAvailableImage(baseImage);

const { os, arch, imageId, instanceType, tags, sshKeys, preemptible } = options;

/** @type {AwsImage} */

let image;

if (imageId) {

image = await aws.getAvailableImage(imageId);

} else {

image = await aws.getBaseImage(options);

}

const { ImageId, Name, RootDeviceName, BlockDeviceMappings } = image;

// console.table({ os, arch, instanceType, Name, ImageId });

const blockDeviceMappings = BlockDeviceMappings.map(device => {

const { DeviceName } = device;

if (DeviceName === RootDeviceName) {

return {

...device,

Ebs: {

VolumeSize: getDiskSize(options),

},

};

}

return device;

});

const username = getUsernameForDistro(Name);

// Only include minimal cloud-init for SSH access

let userData = getUserData({ ...options, username });

if (os === "windows") {

userData = `<powershell>${userData}</powershell><powershellArguments>-ExecutionPolicy Unrestricted -NoProfile -NonInteractive</powershellArguments><persist>true</persist>`;

}

let tagSpecification = [];

if (tags) {

tagSpecification = ["instance", "volume"].map(resourceType => {

return {

ResourceType: resourceType,

Tags: Object.entries(tags).map(([Key, Value]) => ({ Key, Value: String(Value) })),

};

});

}

/** @type {string | undefined} */

let keyName, keyPath;

if (os === "windows") {

const sshKey = sshKeys.find(({ privatePath }) => existsSync(privatePath));

if (sshKey) {

const { publicKey, privatePath } = sshKey;

const { KeyName } = await aws.importKeyPair(publicKey);

keyName = KeyName;

keyPath = privatePath;

}

}

let marketOptions;

if (preemptible) {

marketOptions = JSON.stringify({

MarketType: "spot",

SpotOptions: {

InstanceInterruptionBehavior: "terminate",

SpotInstanceType: "one-time",

},

});

}

// Attach IAM instance profile for CI builds to enable S3 build cache access

let iamInstanceProfile;

if (options.ci) {

iamInstanceProfile = JSON.stringify({ Name: "buildkite-build-agent" });

}

const [instance] = await aws.runInstances({

["image-id"]: ImageId,

["instance-type"]: instanceType || (arch === "aarch64" ? "t4g.large" : "t3.large"),

["user-data"]: userData,

["block-device-mappings"]: JSON.stringify(blockDeviceMappings),

["metadata-options"]: JSON.stringify({

"HttpTokens": "optional",

"HttpEndpoint": "enabled",

"HttpProtocolIpv6": "enabled",

"InstanceMetadataTags": "enabled",

}),

["tag-specifications"]: JSON.stringify(tagSpecification),

["key-name"]: keyName,

["instance-market-options"]: marketOptions,

["iam-instance-profile"]: iamInstanceProfile,

});

const machine = aws.toMachine(instance, { ...options, username, keyPath });

await setupUserData(machine, options);

return machine;

let { InstanceId, ImageId, InstanceType, Placement, PublicIpAddress } = instance;

const connect = async () => {

if (!PublicIpAddress) {

await aws.waitInstances("instance-running", InstanceId);

const [{ PublicIpAddress: IpAddress }] = await aws.describeInstances({

["instance-id"]: InstanceId,

});

PublicIpAddress = IpAddress;

}

const { username, sshKeys } = options;

const identityPaths = sshKeys

?.filter(({ privatePath }) => existsSync(privatePath))

?.map(({ privatePath }) => privatePath);

return { hostname: PublicIpAddress, username, identityPaths };

};

const waitForSsh = async () => {

const connectOptions = await connect();

const { hostname, username, identityPaths } = connectOptions;

// Try to connect until it succeeds

for (let i = 0; i < 30; i++) {

try {

await spawnSshSafe({

hostname,

username,

identityPaths,

command: ["true"],

});

return;

} catch (error) {

if (i === 29) {

throw error;

}

await new Promise(resolve => setTimeout(resolve, 5000));

}

}

};

const spawn = async (command, options) => {

const connectOptions = await connect();

return spawnSsh({ ...connectOptions, command }, options);

};

const spawnSafe = async (command, options) => {

const connectOptions = await connect();

return spawnSshSafe({ ...connectOptions, command }, options);

};

const rdp = async () => {

const { keyPath } = options;

const { hostname, username } = await connect();

const password = await aws.getPasswordData(InstanceId, keyPath, { wait: true });

return { hostname, username, password };

};

const attach = async () => {

const connectOptions = await connect();

await spawnSshSafe({ ...connectOptions });

};

const upload = async (source, destination) => {

const connectOptions = await connect();

await spawnScp({ ...connectOptions, source, destination });

};

const snapshot = async name => {

await aws.stopInstances(InstanceId);

await aws.waitInstances("instance-stopped", InstanceId);

const imageId = await aws.createImage({

["instance-id"]: InstanceId,

["name"]: name || `${InstanceId}-snapshot-${Date.now()}`,

});

await aws.waitImage("image-available", imageId);

return imageId;

};

const terminate = async () => {

await aws.terminateInstances(InstanceId);

};

return {

cloud: "aws",

id: InstanceId,

imageId: ImageId,

instanceType: InstanceType,

region: Placement?.AvailabilityZone,

get publicIp() {

return PublicIpAddress;

},

spawn,

spawnSafe,

upload,

attach,

rdp,

snapshot,

waitForSsh,

close: terminate,

[Symbol.asyncDispose]: terminate,

};

const { os, userData } = cloudInit;

// For Windows, use PowerShell script

if (os === "windows") {

return getWindowsStartupScript(cloudInit);

}

// For Linux, just set up SSH access

return getCloudInit(cloudInit);

const username = cloudInit["username"] || "root";

const password = cloudInit["password"] || crypto.randomUUID();

const authorizedKeys = cloudInit["sshKeys"]?.map(({ publicKey }) => publicKey) || [];

let sftpPath = "/usr/lib/openssh/sftp-server";

let shell = "/bin/bash";

switch (cloudInit["distro"]) {

case "alpine":

sftpPath = "/usr/lib/ssh/sftp-server";

break;

case "amazonlinux":

case "rhel":

case "centos":

sftpPath = "/usr/libexec/openssh/sftp-server";

break;

}

switch (cloudInit["os"]) {

case "linux":

case "windows":

// handled above

break;

default:

throw new Error(`Unsupported os: ${cloudInit["os"]}`);

}

let users;

if (username === "root") {

users = [`root:${password}`];

} else {

users = [`root:${password}`, `${username}:${password}`];

}

// https://cloudinit.readthedocs.io/en/stable/

return `#cloud-config

const { sshKeys } = cloudInit;

const authorizedKeys = sshKeys.map(({ publicKey }) => publicKey);

return `

const { os, diskSizeGb } = options;

if (diskSizeGb) {

return diskSizeGb;

}

// After Visual Studio and dependencies are installed,

// there is ~50GB of used disk space.

if (os === "windows") {

return 60;

}

return 40;

const sshKeyGen = which("ssh-keygen", { required: true });

const sshAdd = which("ssh-add", { required: true });

const sshPath = join(homedir(), ".ssh");

mkdir(sshPath);

const filename = `id_rsa_${crypto.randomUUID()}`;

const privatePath = join(sshPath, filename);

const publicPath = join(sshPath, `${filename}.pub`);

spawnSyncSafe([sshKeyGen, "-t", "rsa", "-b", "4096", "-f", privatePath, "-N", ""], { stdio: "inherit" });

if (!existsSync(privatePath) || !existsSync(publicPath)) {

throw new Error(`Failed to generate SSH key: ${privatePath} / ${publicPath}`);

}

if (isWindows) {

spawnSyncSafe([sshAdd, privatePath], { stdio: "inherit" });

} else {

const sshAgent = which("ssh-agent");

if (sshAgent) {

spawnSyncSafe(["sh", "-c", `eval $(${sshAgent} -s) && ${sshAdd} ${privatePath}`], { stdio: "inherit" });

}

}

return {

privatePath,

publicPath,

get publicKey() {

return readFile(publicPath, { cache: true });

},

};

const homePath = homedir();

const sshPath = join(homePath, ".ssh");

/** @type {SshKey[]} */

const sshKeys = [];

if (existsSync(sshPath)) {

const sshFiles = readdirSync(sshPath, { withFileTypes: true, encoding: "utf-8" });

const publicPaths = sshFiles

.filter(entry => entry.isFile() && entry.name.endsWith(".pub"))