Improve CI caching (#416) · patrickgwsmith/code-server@dcf409a · GitHub
Skip to content

Commit dcf409a

Browse files
kylecarbscode-asher
kylecarbs
authored and
code-asher
committed
Improve CI caching (coder#416)
* Adjust linux distro to ubuntu 14.04 * Cache lib directory for speedy builds * Fix path linking for default extensions * Update reset * Reset to head * Improve caching * Still run yarn in CI * Update yarn before install * Increase cache timeout * Install vscode from vstar * Undo data-dir changes to CLI, add back clean, remove unused CI func * Remove additional flags added * Remove unused dependency * Reset vscode install dir so patching always works
1 parent 2683b7c commit dcf409a

7 files changed

Lines changed: 145 additions & 84 deletions

File tree

‎.gitignore‎

Lines changed: 1 addition & 0 deletions

‎.travis.yml‎

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -6,11 +6,12 @@ env:
66
matrix:
77
include:
88
- os: linux
9-
dist: ubuntu
9+
dist: trusty
1010
- os: osx
1111
before_install:
1212
- if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then sudo apt-get install libxkbfile-dev
1313
libsecret-1-dev; fi
14+
- npm install -g yarn@1.12.3
1415
script:
1516
- scripts/build.sh
1617
before_deploy:
@@ -35,4 +36,8 @@ deploy:
3536
on:
3637
repo: codercom/code-server
3738
branch: master
38-
cache: yarn
39+
cache:
40+
yarn: true
41+
timeout: 1000
42+
directories:
43+
- .cache

‎build/tasks.ts‎

Lines changed: 34 additions & 80 deletions
Original file line numberDiff line numberDiff line change
@@ -4,21 +4,22 @@ import * as fse from "fs-extra";
44
import * as os from "os";
55
import * as path from "path";
66
import * as zlib from "zlib";
7+
import * as https from "https";
8+
import * as tar from "tar";
79

810
const isWin = os.platform() === "win32";
911
const libPath = path.join(__dirname, "../lib");
1012
const vscodePath = path.join(libPath, "vscode");
13+
const defaultExtensionsPath = path.join(libPath, "extensions");
1114
const pkgsPath = path.join(__dirname, "../packages");
12-
const defaultExtensionsPath = path.join(libPath, "VSCode-linux-x64/resources/app/extensions");
1315
const vscodeVersion = process.env.VSCODE_VERSION || "1.32.0";
16+
const vsSourceUrl = `https://codesrv-ci.cdr.sh/vstar-${vscodeVersion}.tar.gz`;
1417

1518
const buildServerBinary = register("build:server:binary", async (runner) => {
1619
await ensureInstalled();
17-
await copyForDefaultExtensions();
1820
await Promise.all([
1921
buildBootstrapFork(),
2022
buildWeb(),
21-
buildDefaultExtensions(),
2223
buildServerBundle(),
2324
buildAppBrowser(),
2425
]);
@@ -129,97 +130,50 @@ const buildWeb = register("build:web", async (runner) => {
129130
await runner.execute(isWin ? "npm.cmd" : "npm", ["run", "build"]);
130131
});
131132

132-
const extDirPath = path.join("lib", "vscode-default-extensions");
133-
const copyForDefaultExtensions = register("build:copy-vscode", async (runner) => {
134-
if (!fs.existsSync(defaultExtensionsPath)) {
135-
await ensureClean();
136-
await ensureInstalled();
137-
await new Promise((resolve, reject): void => {
138-
fse.remove(extDirPath, (err) => {
139-
if (err) {
140-
return reject(err);
141-
}
142-
143-
resolve();
144-
});
145-
});
146-
await new Promise((resolve, reject): void => {
147-
fse.copy(vscodePath, extDirPath, (err) => {
148-
if (err) {
149-
return reject(err);
150-
}
151-
152-
resolve();
153-
});
154-
});
155-
}
156-
});
157-
158-
const buildDefaultExtensions = register("build:default-extensions", async (runner) => {
159-
if (!fs.existsSync(defaultExtensionsPath)) {
160-
await copyForDefaultExtensions();
161-
runner.cwd = extDirPath;
162-
const resp = await runner.execute(isWin ? "npx.cmd" : "npx", [isWin ? "gulp.cmd" : "gulp", "vscode-linux-x64", "--max-old-space-size=32384"]);
163-
if (resp.exitCode !== 0) {
164-
throw new Error(`Failed to build default extensions: ${resp.stderr}`);
165-
}
166-
}
167-
});
168-
169133
const ensureInstalled = register("vscode:install", async (runner) => {
170-
await ensureCloned();
134+
runner.cwd = libPath;
171135

172-
runner.cwd = vscodePath;
173-
const install = await runner.execute(isWin ? "yarn.cmd" : "yarn", []);
174-
if (install.exitCode !== 0) {
175-
throw new Error(`Failed to install vscode dependencies: ${install.stderr}`);
176-
}
177-
});
136+
if (fs.existsSync(vscodePath) && fs.existsSync(defaultExtensionsPath)) {
137+
const pkgVersion = JSON.parse(fs.readFileSync(path.join(vscodePath, "package.json")).toString("utf8")).version;
138+
if (pkgVersion === vscodeVersion) {
139+
runner.cwd = vscodePath;
178140

179-
const ensureCloned = register("vscode:clone", async (runner) => {
180-
if (fs.existsSync(vscodePath)) {
181-
await ensureClean();
182-
} else {
183-
fse.mkdirpSync(libPath);
184-
runner.cwd = libPath;
185-
const clone = await runner.execute("git", ["clone", "https://github.com/microsoft/vscode", "--branch", vscodeVersion, "--single-branch", "--depth=1"]);
186-
if (clone.exitCode !== 0) {
187-
throw new Error(`Failed to clone: ${clone.exitCode}`);
141+
const reset = await runner.execute("git", ["reset", "--hard"]);
142+
if (reset.exitCode !== 0) {
143+
throw new Error(`Failed to clean git repository: ${reset.stderr}`);
144+
}
145+
146+
return;
188147
}
189148
}
190149

191-
runner.cwd = vscodePath;
192-
const checkout = await runner.execute("git", ["checkout", vscodeVersion]);
193-
if (checkout.exitCode !== 0) {
194-
throw new Error(`Failed to checkout: ${checkout.stderr}`);
195-
}
196-
});
150+
fse.removeSync(libPath);
151+
fse.mkdirpSync(libPath);
197152

198-
const ensureClean = register("vscode:clean", async (runner) => {
199-
runner.cwd = vscodePath;
153+
await new Promise<void>((resolve, reject): void => {
154+
https.get(vsSourceUrl, (res) => {
155+
if (res.statusCode !== 200) {
156+
return reject(res.statusMessage);
157+
}
200158

201-
const status = await runner.execute("git", ["status", "--porcelain"]);
202-
if (status.stdout.trim() !== "") {
203-
const clean = await runner.execute("git", ["clean", "-f", "-d", "-X"]);
204-
if (clean.exitCode !== 0) {
205-
throw new Error(`Failed to clean git repository: ${clean.stderr}`);
206-
}
207-
const removeUnstaged = await runner.execute("git", ["checkout", "--", "."]);
208-
if (removeUnstaged.exitCode !== 0) {
209-
throw new Error(`Failed to remove unstaged files: ${removeUnstaged.stderr}`);
210-
}
211-
}
212-
const fetch = await runner.execute("git", ["fetch", "--prune"]);
213-
if (fetch.exitCode !== 0) {
214-
throw new Error(`Failed to fetch latest changes: ${fetch.stderr}`);
215-
}
159+
res.pipe(tar.x({
160+
C: libPath,
161+
}).on("finish", () => {
162+
resolve();
163+
}).on("error", (err: Error) => {
164+
reject(err);
165+
}));
166+
}).on("error", (err) => {
167+
reject(err);
168+
});
169+
});
216170
});
217171

218172
const ensurePatched = register("vscode:patch", async (runner) => {
219173
if (!fs.existsSync(vscodePath)) {
220174
throw new Error("vscode must be cloned to patch");
221175
}
222-
await ensureClean();
176+
await ensureInstalled();
223177

224178
runner.cwd = vscodePath;
225179
const patchPath = path.join(__dirname, "../scripts/vscode.patch");

‎package.json‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,9 @@
1515
"devDependencies": {
1616
"@types/fs-extra": "^5.0.4",
1717
"@types/node": "^10.12.18",
18+
"@types/tar": "^4.0.0",
1819
"@types/trash": "^4.3.1",
20+
"cache-loader": "^2.0.1",
1921
"cross-env": "^5.2.0",
2022
"crypto-browserify": "^3.12.0",
2123
"css-loader": "^2.1.0",
@@ -34,6 +36,8 @@
3436
"sass-loader": "^7.1.0",
3537
"string-replace-loader": "^2.1.1",
3638
"style-loader": "^0.23.1",
39+
"tar": "^4.4.8",
40+
"terser-webpack-plugin": "^1.2.3",
3741
"ts-loader": "^5.3.3",
3842
"ts-node": "^7.0.1",
3943
"tsconfig-paths": "^3.8.0",

‎scripts/vstar.sh‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,24 @@
1+
#!/bin/bash
2+
set -euxo pipefail
3+
4+
# Builds a tarfile containing vscode sourcefiles neccessary for CI.
5+
# Done outside the CI and uploaded to object storage to reduce CI time.
6+
7+
branch=1.32.0
8+
dir=/tmp/vstar
9+
outfile=/tmp/vstar-$branch.tar.gz
10+
rm -rf $dir
11+
mkdir -p $dir
12+
13+
cd $dir
14+
git clone https://github.com/microsoft/vscode --branch $branch --single-branch --depth=1
15+
cd vscode
16+
17+
yarn
18+
19+
npx gulp vscode-linux-x64 --max-old-space-size=32384
20+
rm -rf extensions build out* test
21+
cd ..
22+
mv *-x64/resources/app/extensions ./extensions
23+
rm -rf *-x64
24+
tar -czvf $outfile .

‎scripts/webpack.general.config.js‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@ const os = require("os");
33
const environment = process.env.NODE_ENV || "development";
44
const HappyPack = require("happypack");
55
const webpack = require("webpack");
6+
const TerserPlugin = require("terser-webpack-plugin");
67

78
const root = path.join(__dirname, "..");
89

@@ -107,6 +108,11 @@ module.exports = (options = {}) => ({
107108
id: "ts",
108109
threads: Math.max(os.cpus().length - 1, 1),
109110
loaders: [{
111+
path: "cache-loader",
112+
query: {
113+
cacheDirectory: path.join(__dirname, "..", ".cache"),
114+
},
115+
}, {
110116
path: "ts-loader",
111117
query: {
112118
happyPackMode: true,
@@ -121,6 +127,14 @@ module.exports = (options = {}) => ({
121127
"process.env.VERSION": `"${process.env.VERSION || ""}"`,
122128
}),
123129
],
130+
optimization: {
131+
minimizer: [
132+
new TerserPlugin({
133+
cache: path.join(__dirname, "..", ".cache", "terser"),
134+
parallel: true,
135+
}),
136+
],
137+
},
124138
stats: {
125139
all: false, // Fallback for options not defined.
126140
errors: true,

‎yarn.lock‎

Lines changed: 61 additions & 2 deletions

0 commit comments

Comments
 (0)