15 Commits

Author SHA1 Message Date
Arpad Borsos
bd4d2a7017 1.0.2 2020-09-29 12:30:45 +02:00
Arpad Borsos
d38127a85b Improve target pruning
fixes #1
2020-09-29 12:30:19 +02:00
Arpad Borsos
a4a1d8e7a6 1.0.1 2020-09-28 13:06:41 +02:00
Arpad Borsos
33677a20f2 add changelog 2020-09-28 12:53:56 +02:00
Arpad Borsos
1d1bff80c5 update readme 2020-09-28 12:51:40 +02:00
Arpad Borsos
08ca2ff969 make macos workaround silent 2020-09-28 12:46:36 +02:00
Arpad Borsos
ef89c3a8eb typo 2020-09-28 12:30:40 +02:00
Arpad Borsos
d45cd2b045 rebuild 2020-09-28 12:26:12 +02:00
Arpad Borsos
271ff4b692 log individual timings 2020-09-28 12:26:11 +02:00
Arpad Borsos
a6b59fa340 clean up exports 2020-09-28 12:14:11 +02:00
Arpad Borsos
e0c07d2a65 work around macos cache corruption 2020-09-28 12:08:11 +02:00
Arpad Borsos
06ff70612d remove git-db for now 2020-09-28 12:06:51 +02:00
Arpad Borsos
1304a2ec8d add ability to version caches 2020-09-28 12:00:58 +02:00
Arpad Borsos
cfcc373039 improve logging 2020-09-28 11:54:24 +02:00
Arpad Borsos
8902a8fc6c collect packages with --all-features 2020-09-28 11:44:06 +02:00
9 changed files with 580 additions and 443 deletions

12
CHANGELOG.md Normal file
View File

@@ -0,0 +1,12 @@
# Changelog
## 1.0.2
- Dont prune targets that have a different name from the crate, but do prune targets from the workspace.
## 1.0.1
- Improved logging output.
- Make sure to consider `all-features` dependencies when pruning.
- Work around macOS cache corruption.
- Remove git-db cache for now.

View File

@@ -30,20 +30,16 @@ assumption is that we will likely recompile the own crate(s) anyway.
It also separates the cache into 4 groups, each treated differently: It also separates the cache into 4 groups, each treated differently:
- Index: `~/.cargo/registry/index/<registry>`: - Registry Index: `~/.cargo/registry/index/<registry>`:
This is always restored from its latest snapshot, and persisted based on the This is always restored from its latest snapshot, and persisted based on the
most recent revision. most recent revision.
- Registry / Cache: `~/.cargo/registry/cache/<registry>`: - Registry Cache: `~/.cargo/registry/cache/<registry>`:
Automatically keyed by the lockfile/toml hash, and is being pruned to only Automatically keyed by the lockfile/toml hash, and is being pruned to only
persist the dependencies that are being used. persist the dependencies that are being used.
- Registry / Git: `~/.cargo/registry/git/<registry>`:
Automatically keyed by the lockfile/toml hash. Pruning is still TODO.
- target: `./target` - target: `./target`
Automatically keyed by the lockfile/toml hash, and is being pruned to only Automatically keyed by the lockfile/toml hash, and is being pruned to only

301
dist/restore/index.js vendored
View File

@@ -54598,147 +54598,170 @@ var external_path_ = __webpack_require__(5622);
var external_path_default = /*#__PURE__*/__webpack_require__.n(external_path_); var external_path_default = /*#__PURE__*/__webpack_require__.n(external_path_);
// CONCATENATED MODULE: ./src/common.ts // CONCATENATED MODULE: ./src/common.ts
var __asyncValues = (undefined && undefined.__asyncValues) || function (o) { var __asyncValues = (undefined && undefined.__asyncValues) || function (o) {
if (!Symbol.asyncIterator) throw new TypeError("Symbol.asyncIterator is not defined."); if (!Symbol.asyncIterator) throw new TypeError("Symbol.asyncIterator is not defined.");
var m = o[Symbol.asyncIterator], i; var m = o[Symbol.asyncIterator], i;
return m ? m.call(o) : (o = typeof __values === "function" ? __values(o) : o[Symbol.iterator](), i = {}, verb("next"), verb("throw"), verb("return"), i[Symbol.asyncIterator] = function () { return this; }, i); return m ? m.call(o) : (o = typeof __values === "function" ? __values(o) : o[Symbol.iterator](), i = {}, verb("next"), verb("throw"), verb("return"), i[Symbol.asyncIterator] = function () { return this; }, i);
function verb(n) { i[n] = o[n] && function (v) { return new Promise(function (resolve, reject) { v = o[n](v), settle(resolve, reject, v.done, v.value); }); }; } function verb(n) { i[n] = o[n] && function (v) { return new Promise(function (resolve, reject) { v = o[n](v), settle(resolve, reject, v.done, v.value); }); }; }
function settle(resolve, reject, d, v) { Promise.resolve(v).then(function(v) { resolve({ value: v, done: d }); }, reject); } function settle(resolve, reject, d, v) { Promise.resolve(v).then(function(v) { resolve({ value: v, done: d }); }, reject); }
}; };
const home = external_os_default().homedir(); const home = external_os_default().homedir();
const paths = { const paths = {
index: external_path_default().join(home, ".cargo/registry/index"), index: external_path_default().join(home, ".cargo/registry/index"),
cache: external_path_default().join(home, ".cargo/registry/cache"), cache: external_path_default().join(home, ".cargo/registry/cache"),
git: external_path_default().join(home, ".cargo/git/db"), git: external_path_default().join(home, ".cargo/git/db"),
target: "target", target: "target",
}; };
const RefKey = "GITHUB_REF"; const RefKey = "GITHUB_REF";
function isValidEvent() { function isValidEvent() {
return RefKey in process.env && Boolean(process.env[RefKey]); return RefKey in process.env && Boolean(process.env[RefKey]);
} }
async function getCaches() { async function getCaches() {
const rustKey = await getRustKey(); const rustKey = await getRustKey();
let lockHash = core.getState("lockHash"); let lockHash = core.getState("lockHash");
if (!lockHash) { if (!lockHash) {
lockHash = await getLockfileHash(); lockHash = await getLockfileHash();
core.saveState("lockHash", lockHash); core.saveState("lockHash", lockHash);
} }
let targetKey = core.getInput("key"); let targetKey = core.getInput("key");
if (targetKey) { if (targetKey) {
targetKey = `${targetKey}-`; targetKey = `${targetKey}-`;
} }
return { const registryIndex = `v0-registry-index`;
index: { path: paths.index, key: "registry-index-XXX", restoreKeys: ["registry-index"] }, const registryCache = `v0-registry-cache`;
cache: { path: paths.cache, key: `registry-cache-${lockHash}`, restoreKeys: ["registry-cache"] }, const target = `v0-target-${targetKey}${rustKey}`;
git: { path: paths.git, key: "git-db" }, return {
target: { index: {
path: paths.target, name: "Registry Index",
key: `target-${targetKey}${rustKey}-${lockHash}`, path: paths.index,
restoreKeys: [`target-${targetKey}${rustKey}`], key: `${registryIndex}-`,
}, restoreKeys: [registryIndex],
}; },
} cache: {
async function getRustKey() { name: "Registry Cache",
const rustc = await getRustVersion(); path: paths.cache,
return `${rustc.release}-${rustc.host}-${rustc["commit-hash"]}`; key: `${registryCache}-${lockHash}`,
} restoreKeys: [registryCache],
async function getRustVersion() { },
const stdout = await getCmdOutput("rustc", ["-vV"]); // git: {
let splits = stdout // name: "Git Dependencies",
.split(/[\n\r]+/) // path: paths.git,
.filter(Boolean) // key: "git-db",
.map((s) => s.split(":").map((s) => s.trim())) // },
.filter((s) => s.length === 2); target: {
return Object.fromEntries(splits); name: "Target",
} path: paths.target,
async function getCmdOutput(cmd, args = [], options = {}) { key: `${target}-${lockHash}`,
let stdout = ""; restoreKeys: [target],
await exec.exec(cmd, args, Object.assign({ silent: true, listeners: { },
stdout(data) { };
stdout += data.toString(); }
}, async function getRustKey() {
} }, options)); const rustc = await getRustVersion();
return stdout; return `${rustc.release}-${rustc.host}-${rustc["commit-hash"]}`;
} }
async function getRegistryName() { async function getRustVersion() {
const globber = await glob.create(`${paths.index}/**/.last-updated`, { followSymbolicLinks: false }); const stdout = await getCmdOutput("rustc", ["-vV"]);
const files = await globber.glob(); let splits = stdout
if (files.length > 1) { .split(/[\n\r]+/)
core.debug(`got multiple registries: "${files.join('", "')}"`); .filter(Boolean)
} .map((s) => s.split(":").map((s) => s.trim()))
const first = files.shift(); .filter((s) => s.length === 2);
if (!first) { return Object.fromEntries(splits);
return; }
} async function getCmdOutput(cmd, args = [], options = {}) {
return external_path_default().basename(external_path_default().dirname(first)); let stdout = "";
} await exec.exec(cmd, args, Object.assign({ silent: true, listeners: {
async function getLockfileHash() { stdout(data) {
var e_1, _a; stdout += data.toString();
const globber = await glob.create("**/Cargo.toml\n**/Cargo.lock", { followSymbolicLinks: false }); },
const files = await globber.glob(); } }, options));
files.sort((a, b) => a.localeCompare(b)); return stdout;
const hasher = external_crypto_default().createHash("sha1"); }
for (const file of files) { async function getRegistryName() {
try { const globber = await glob.create(`${paths.index}/**/.last-updated`, { followSymbolicLinks: false });
for (var _b = (e_1 = void 0, __asyncValues(external_fs_default().createReadStream(file))), _c; _c = await _b.next(), !_c.done;) { const files = await globber.glob();
const chunk = _c.value; if (files.length > 1) {
hasher.update(chunk); core.debug(`got multiple registries: "${files.join('", "')}"`);
} }
} const first = files.shift();
catch (e_1_1) { e_1 = { error: e_1_1 }; } if (!first) {
finally { return;
try { }
if (_c && !_c.done && (_a = _b.return)) await _a.call(_b); return external_path_default().basename(external_path_default().dirname(first));
} }
finally { if (e_1) throw e_1.error; } async function getLockfileHash() {
} var e_1, _a;
} const globber = await glob.create("**/Cargo.toml\n**/Cargo.lock", { followSymbolicLinks: false });
return hasher.digest("hex"); const files = await globber.glob();
} files.sort((a, b) => a.localeCompare(b));
const hasher = external_crypto_default().createHash("sha1");
for (const file of files) {
try {
for (var _b = (e_1 = void 0, __asyncValues(external_fs_default().createReadStream(file))), _c; _c = await _b.next(), !_c.done;) {
const chunk = _c.value;
hasher.update(chunk);
}
}
catch (e_1_1) { e_1 = { error: e_1_1 }; }
finally {
try {
if (_c && !_c.done && (_a = _b.return)) await _a.call(_b);
}
finally { if (e_1) throw e_1.error; }
}
}
return hasher.digest("hex");
}
// CONCATENATED MODULE: ./src/restore.ts // CONCATENATED MODULE: ./src/restore.ts
async function run() { async function run() {
if (!isValidEvent()) { if (!isValidEvent()) {
return; return;
} }
try { try {
core.exportVariable("CARGO_INCREMENTAL", 0); core.exportVariable("CARGO_INCREMENTAL", 0);
const caches = await getCaches(); const caches = await getCaches();
for (const [name, { path, key, restoreKeys }] of Object.entries(caches)) { for (const [type, { name, path, key, restoreKeys }] of Object.entries(caches)) {
try { const start = Date.now();
core.startGroup(`Restoring "${path}" from "${key}"`); core.startGroup(`Restoring ${name}`);
const restoreKey = await cache.restoreCache([path], key, restoreKeys); core.info(`Restoring to path "${path}".`);
if (restoreKey) { core.info(`Using keys:\n ${[key, ...restoreKeys].join("\n ")}`);
core.info(`Restored "${path}" from cache key "${restoreKey}".`); try {
core.saveState(name, restoreKey); const restoreKey = await cache.restoreCache([path], key, restoreKeys);
} if (restoreKey) {
else { core.info(`Restored from cache key "${restoreKey}".`);
core.info("No cache found."); core.saveState(type, restoreKey);
} }
} else {
catch (e) { core.info("No cache found.");
core.info(`[warning] ${e.message}`); }
} }
finally { catch (e) {
core.endGroup(); core.info(`[warning] ${e.message}`);
} }
} const duration = Math.round((Date.now() - start) / 1000);
} if (duration) {
catch (e) { core.info(`Took ${duration}s.`);
core.info(`[warning] ${e.message}`); }
} core.endGroup();
} }
run(); }
catch (e) {
core.info(`[warning] ${e.message}`);
}
}
run();
/***/ }), /***/ }),

570
dist/save/index.js vendored
View File

@@ -54581,285 +54581,327 @@ var exec = __webpack_require__(1514);
// EXTERNAL MODULE: ./node_modules/@actions/glob/lib/glob.js // EXTERNAL MODULE: ./node_modules/@actions/glob/lib/glob.js
var glob = __webpack_require__(8090); var glob = __webpack_require__(8090);
// EXTERNAL MODULE: ./node_modules/@actions/io/lib/io.js
var io = __webpack_require__(7436);
// EXTERNAL MODULE: external "crypto" // EXTERNAL MODULE: external "crypto"
var external_crypto_ = __webpack_require__(6417); var external_crypto_ = __webpack_require__(6417);
var external_crypto_default = /*#__PURE__*/__webpack_require__.n(external_crypto_); var external_crypto_default = /*#__PURE__*/__webpack_require__.n(external_crypto_);
// EXTERNAL MODULE: ./node_modules/@actions/io/lib/io.js
var io = __webpack_require__(7436);
// EXTERNAL MODULE: external "fs" // EXTERNAL MODULE: external "fs"
var external_fs_ = __webpack_require__(5747); var external_fs_ = __webpack_require__(5747);
var external_fs_default = /*#__PURE__*/__webpack_require__.n(external_fs_); var external_fs_default = /*#__PURE__*/__webpack_require__.n(external_fs_);
// EXTERNAL MODULE: external "path"
var external_path_ = __webpack_require__(5622);
var external_path_default = /*#__PURE__*/__webpack_require__.n(external_path_);
// EXTERNAL MODULE: external "os" // EXTERNAL MODULE: external "os"
var external_os_ = __webpack_require__(2087); var external_os_ = __webpack_require__(2087);
var external_os_default = /*#__PURE__*/__webpack_require__.n(external_os_); var external_os_default = /*#__PURE__*/__webpack_require__.n(external_os_);
// EXTERNAL MODULE: external "path"
var external_path_ = __webpack_require__(5622);
var external_path_default = /*#__PURE__*/__webpack_require__.n(external_path_);
// CONCATENATED MODULE: ./src/common.ts // CONCATENATED MODULE: ./src/common.ts
var __asyncValues = (undefined && undefined.__asyncValues) || function (o) { var __asyncValues = (undefined && undefined.__asyncValues) || function (o) {
if (!Symbol.asyncIterator) throw new TypeError("Symbol.asyncIterator is not defined."); if (!Symbol.asyncIterator) throw new TypeError("Symbol.asyncIterator is not defined.");
var m = o[Symbol.asyncIterator], i; var m = o[Symbol.asyncIterator], i;
return m ? m.call(o) : (o = typeof __values === "function" ? __values(o) : o[Symbol.iterator](), i = {}, verb("next"), verb("throw"), verb("return"), i[Symbol.asyncIterator] = function () { return this; }, i); return m ? m.call(o) : (o = typeof __values === "function" ? __values(o) : o[Symbol.iterator](), i = {}, verb("next"), verb("throw"), verb("return"), i[Symbol.asyncIterator] = function () { return this; }, i);
function verb(n) { i[n] = o[n] && function (v) { return new Promise(function (resolve, reject) { v = o[n](v), settle(resolve, reject, v.done, v.value); }); }; } function verb(n) { i[n] = o[n] && function (v) { return new Promise(function (resolve, reject) { v = o[n](v), settle(resolve, reject, v.done, v.value); }); }; }
function settle(resolve, reject, d, v) { Promise.resolve(v).then(function(v) { resolve({ value: v, done: d }); }, reject); } function settle(resolve, reject, d, v) { Promise.resolve(v).then(function(v) { resolve({ value: v, done: d }); }, reject); }
}; };
const home = external_os_default().homedir(); const home = external_os_default().homedir();
const paths = { const paths = {
index: external_path_default().join(home, ".cargo/registry/index"), index: external_path_default().join(home, ".cargo/registry/index"),
cache: external_path_default().join(home, ".cargo/registry/cache"), cache: external_path_default().join(home, ".cargo/registry/cache"),
git: external_path_default().join(home, ".cargo/git/db"), git: external_path_default().join(home, ".cargo/git/db"),
target: "target", target: "target",
}; };
const RefKey = "GITHUB_REF"; const RefKey = "GITHUB_REF";
function isValidEvent() { function isValidEvent() {
return RefKey in process.env && Boolean(process.env[RefKey]); return RefKey in process.env && Boolean(process.env[RefKey]);
} }
async function getCaches() { async function getCaches() {
const rustKey = await getRustKey(); const rustKey = await getRustKey();
let lockHash = core.getState("lockHash"); let lockHash = core.getState("lockHash");
if (!lockHash) { if (!lockHash) {
lockHash = await getLockfileHash(); lockHash = await getLockfileHash();
core.saveState("lockHash", lockHash); core.saveState("lockHash", lockHash);
} }
let targetKey = core.getInput("key"); let targetKey = core.getInput("key");
if (targetKey) { if (targetKey) {
targetKey = `${targetKey}-`; targetKey = `${targetKey}-`;
} }
return { const registryIndex = `v0-registry-index`;
index: { path: paths.index, key: "registry-index-XXX", restoreKeys: ["registry-index"] }, const registryCache = `v0-registry-cache`;
cache: { path: paths.cache, key: `registry-cache-${lockHash}`, restoreKeys: ["registry-cache"] }, const target = `v0-target-${targetKey}${rustKey}`;
git: { path: paths.git, key: "git-db" }, return {
target: { index: {
path: paths.target, name: "Registry Index",
key: `target-${targetKey}${rustKey}-${lockHash}`, path: paths.index,
restoreKeys: [`target-${targetKey}${rustKey}`], key: `${registryIndex}-`,
}, restoreKeys: [registryIndex],
}; },
} cache: {
async function getRustKey() { name: "Registry Cache",
const rustc = await getRustVersion(); path: paths.cache,
return `${rustc.release}-${rustc.host}-${rustc["commit-hash"]}`; key: `${registryCache}-${lockHash}`,
} restoreKeys: [registryCache],
async function getRustVersion() { },
const stdout = await getCmdOutput("rustc", ["-vV"]); // git: {
let splits = stdout // name: "Git Dependencies",
.split(/[\n\r]+/) // path: paths.git,
.filter(Boolean) // key: "git-db",
.map((s) => s.split(":").map((s) => s.trim())) // },
.filter((s) => s.length === 2); target: {
return Object.fromEntries(splits); name: "Target",
} path: paths.target,
async function getCmdOutput(cmd, args = [], options = {}) { key: `${target}-${lockHash}`,
let stdout = ""; restoreKeys: [target],
await exec.exec(cmd, args, Object.assign({ silent: true, listeners: { },
stdout(data) { };
stdout += data.toString(); }
}, async function getRustKey() {
} }, options)); const rustc = await getRustVersion();
return stdout; return `${rustc.release}-${rustc.host}-${rustc["commit-hash"]}`;
} }
async function getRegistryName() { async function getRustVersion() {
const globber = await glob.create(`${paths.index}/**/.last-updated`, { followSymbolicLinks: false }); const stdout = await getCmdOutput("rustc", ["-vV"]);
const files = await globber.glob(); let splits = stdout
if (files.length > 1) { .split(/[\n\r]+/)
core.debug(`got multiple registries: "${files.join('", "')}"`); .filter(Boolean)
} .map((s) => s.split(":").map((s) => s.trim()))
const first = files.shift(); .filter((s) => s.length === 2);
if (!first) { return Object.fromEntries(splits);
return; }
} async function getCmdOutput(cmd, args = [], options = {}) {
return external_path_default().basename(external_path_default().dirname(first)); let stdout = "";
} await exec.exec(cmd, args, Object.assign({ silent: true, listeners: {
async function getLockfileHash() { stdout(data) {
var e_1, _a; stdout += data.toString();
const globber = await glob.create("**/Cargo.toml\n**/Cargo.lock", { followSymbolicLinks: false }); },
const files = await globber.glob(); } }, options));
files.sort((a, b) => a.localeCompare(b)); return stdout;
const hasher = external_crypto_default().createHash("sha1"); }
for (const file of files) { async function getRegistryName() {
try { const globber = await glob.create(`${paths.index}/**/.last-updated`, { followSymbolicLinks: false });
for (var _b = (e_1 = void 0, __asyncValues(external_fs_default().createReadStream(file))), _c; _c = await _b.next(), !_c.done;) { const files = await globber.glob();
const chunk = _c.value; if (files.length > 1) {
hasher.update(chunk); core.debug(`got multiple registries: "${files.join('", "')}"`);
} }
} const first = files.shift();
catch (e_1_1) { e_1 = { error: e_1_1 }; } if (!first) {
finally { return;
try { }
if (_c && !_c.done && (_a = _b.return)) await _a.call(_b); return external_path_default().basename(external_path_default().dirname(first));
} }
finally { if (e_1) throw e_1.error; } async function getLockfileHash() {
} var e_1, _a;
} const globber = await glob.create("**/Cargo.toml\n**/Cargo.lock", { followSymbolicLinks: false });
return hasher.digest("hex"); const files = await globber.glob();
} files.sort((a, b) => a.localeCompare(b));
const hasher = external_crypto_default().createHash("sha1");
for (const file of files) {
try {
for (var _b = (e_1 = void 0, __asyncValues(external_fs_default().createReadStream(file))), _c; _c = await _b.next(), !_c.done;) {
const chunk = _c.value;
hasher.update(chunk);
}
}
catch (e_1_1) { e_1 = { error: e_1_1 }; }
finally {
try {
if (_c && !_c.done && (_a = _b.return)) await _a.call(_b);
}
finally { if (e_1) throw e_1.error; }
}
}
return hasher.digest("hex");
}
// CONCATENATED MODULE: ./src/save.ts // CONCATENATED MODULE: ./src/save.ts
var save_asyncValues = (undefined && undefined.__asyncValues) || function (o) { var save_asyncValues = (undefined && undefined.__asyncValues) || function (o) {
if (!Symbol.asyncIterator) throw new TypeError("Symbol.asyncIterator is not defined."); if (!Symbol.asyncIterator) throw new TypeError("Symbol.asyncIterator is not defined.");
var m = o[Symbol.asyncIterator], i; var m = o[Symbol.asyncIterator], i;
return m ? m.call(o) : (o = typeof __values === "function" ? __values(o) : o[Symbol.iterator](), i = {}, verb("next"), verb("throw"), verb("return"), i[Symbol.asyncIterator] = function () { return this; }, i); return m ? m.call(o) : (o = typeof __values === "function" ? __values(o) : o[Symbol.iterator](), i = {}, verb("next"), verb("throw"), verb("return"), i[Symbol.asyncIterator] = function () { return this; }, i);
function verb(n) { i[n] = o[n] && function (v) { return new Promise(function (resolve, reject) { v = o[n](v), settle(resolve, reject, v.done, v.value); }); }; } function verb(n) { i[n] = o[n] && function (v) { return new Promise(function (resolve, reject) { v = o[n](v), settle(resolve, reject, v.done, v.value); }); }; }
function settle(resolve, reject, d, v) { Promise.resolve(v).then(function(v) { resolve({ value: v, done: d }); }, reject); } function settle(resolve, reject, d, v) { Promise.resolve(v).then(function(v) { resolve({ value: v, done: d }); }, reject); }
}; };
async function run() {
if (!isValidEvent()) { async function run() {
//return; if (!isValidEvent()) {
} return;
try { }
const caches = await getCaches(); try {
const registryName = await getRegistryName(); const caches = await getCaches();
const packages = await getPackages(); const registryName = await getRegistryName();
await pruneTarget(packages); const packages = await getPackages();
if (registryName) { // TODO: remove this once https://github.com/actions/toolkit/pull/553 lands
// save the index based on its revision await macOsWorkaround();
const indexRef = await getIndexRef(registryName); await pruneTarget(packages);
caches.index.key = `registry-index-${indexRef}`; if (registryName) {
await io.rmRF(external_path_default().join(paths.index, registryName, ".cache")); // save the index based on its revision
await pruneRegistryCache(registryName, packages); const indexRef = await getIndexRef(registryName);
} caches.index.key += indexRef;
else { await io.rmRF(external_path_default().join(paths.index, registryName, ".cache"));
delete caches.index; await pruneRegistryCache(registryName, packages);
delete caches.cache; }
} else {
for (const [name, { path, key }] of Object.entries(caches)) { delete caches.index;
if (core.getState(name) === key) { delete caches.cache;
core.info(`Cache for "${path}" up-to-date.`); }
continue; for (const [type, { name, path, key }] of Object.entries(caches)) {
} if (core.getState(type) === key) {
try { core.info(`${name} up-to-date.`);
core.startGroup(`Saving "${path}" to cache key "${key}"…`); continue;
if (await cache.saveCache([path], key)) { }
core.info(`Saved "${path}" to cache key "${key}".`); const start = Date.now();
} core.startGroup(`Saving ${name}`);
} core.info(`Saving path "${path}".`);
catch (e) { core.info(`Using key "${key}".`);
core.info(`[warning] ${e.message}`); try {
} await cache.saveCache([path], key);
finally { }
core.endGroup(); catch (e) {
} core.info(`[warning] ${e.message}`);
} }
} const duration = Math.round((Date.now() - start) / 1000);
catch (e) { if (duration) {
core.info(`[warning] ${e.message}`); core.info(`Took ${duration}s.`);
} }
} core.endGroup();
run(); }
async function getIndexRef(registryName) { }
const cwd = external_path_default().join(paths.index, registryName); catch (e) {
return (await getCmdOutput("git", ["rev-parse", "--short", "origin/master"], { cwd })).trim(); core.info(`[warning] ${e.message}`);
} }
async function getPackages() { }
const meta = JSON.parse(await getCmdOutput("cargo", ["metadata", "--format-version", "1"])); run();
return meta.packages.map(({ name, version }) => ({ name, version })); async function getIndexRef(registryName) {
} const cwd = external_path_default().join(paths.index, registryName);
async function pruneRegistryCache(registryName, packages) { return (await getCmdOutput("git", ["rev-parse", "--short", "origin/master"], { cwd })).trim();
var e_1, _a; }
const pkgSet = new Set(packages.map((p) => `${p.name}-${p.version}.crate`)); async function getPackages() {
const dir = await external_fs_default().promises.opendir(external_path_default().join(paths.cache, registryName)); const cwd = process.cwd();
try { const meta = JSON.parse(await getCmdOutput("cargo", ["metadata", "--all-features", "--format-version", "1"]));
for (var dir_1 = save_asyncValues(dir), dir_1_1; dir_1_1 = await dir_1.next(), !dir_1_1.done;) { return meta.packages
const dirent = dir_1_1.value; .filter((p) => !p.manifest_path.startsWith(cwd))
if (dirent.isFile() && !pkgSet.has(dirent.name)) { .map((p) => {
const fileName = external_path_default().join(dir.path, dirent.name); const targets = p.targets.filter((t) => t.kind[0] === "lib").map((t) => t.name);
await external_fs_default().promises.unlink(fileName); return { name: p.name, version: p.version, targets };
core.debug(`deleting "${fileName}"`); });
} }
} async function pruneRegistryCache(registryName, packages) {
} var e_1, _a;
catch (e_1_1) { e_1 = { error: e_1_1 }; } const pkgSet = new Set(packages.map((p) => `${p.name}-${p.version}.crate`));
finally { const dir = await external_fs_default().promises.opendir(external_path_default().join(paths.cache, registryName));
try { try {
if (dir_1_1 && !dir_1_1.done && (_a = dir_1.return)) await _a.call(dir_1); for (var dir_1 = save_asyncValues(dir), dir_1_1; dir_1_1 = await dir_1.next(), !dir_1_1.done;) {
} const dirent = dir_1_1.value;
finally { if (e_1) throw e_1.error; } if (dirent.isFile() && !pkgSet.has(dirent.name)) {
} const fileName = external_path_default().join(dir.path, dirent.name);
} await external_fs_default().promises.unlink(fileName);
async function pruneTarget(packages) { core.debug(`deleting "${fileName}"`);
var e_2, _a; }
await external_fs_default().promises.unlink("./target/.rustc_info.json"); }
await io.rmRF("./target/debug/examples"); }
await io.rmRF("./target/debug/incremental"); catch (e_1_1) { e_1 = { error: e_1_1 }; }
let dir; finally {
// remove all *files* from debug try {
dir = await external_fs_default().promises.opendir("./target/debug"); if (dir_1_1 && !dir_1_1.done && (_a = dir_1.return)) await _a.call(dir_1);
try { }
for (var dir_2 = save_asyncValues(dir), dir_2_1; dir_2_1 = await dir_2.next(), !dir_2_1.done;) { finally { if (e_1) throw e_1.error; }
const dirent = dir_2_1.value; }
if (dirent.isFile()) { }
const fileName = external_path_default().join(dir.path, dirent.name); async function pruneTarget(packages) {
await external_fs_default().promises.unlink(fileName); var e_2, _a;
} await external_fs_default().promises.unlink("./target/.rustc_info.json");
} await io.rmRF("./target/debug/examples");
} await io.rmRF("./target/debug/incremental");
catch (e_2_1) { e_2 = { error: e_2_1 }; } let dir;
finally { // remove all *files* from debug
try { dir = await external_fs_default().promises.opendir("./target/debug");
if (dir_2_1 && !dir_2_1.done && (_a = dir_2.return)) await _a.call(dir_2); try {
} for (var dir_2 = save_asyncValues(dir), dir_2_1; dir_2_1 = await dir_2.next(), !dir_2_1.done;) {
finally { if (e_2) throw e_2.error; } const dirent = dir_2_1.value;
} if (dirent.isFile()) {
const keepPkg = new Set(packages.map((p) => p.name)); const fileName = external_path_default().join(dir.path, dirent.name);
await rmExcept("./target/debug/build", keepPkg); await external_fs_default().promises.unlink(fileName);
await rmExcept("./target/debug/.fingerprint", keepPkg); }
const keepDeps = new Set(packages.flatMap((p) => { }
const name = p.name.replace(/-/g, "_"); }
return [name, `lib${name}`]; catch (e_2_1) { e_2 = { error: e_2_1 }; }
})); finally {
await rmExcept("./target/debug/deps", keepDeps); try {
} if (dir_2_1 && !dir_2_1.done && (_a = dir_2.return)) await _a.call(dir_2);
const twoWeeks = 14 * 24 * 3600 * 1000; }
async function rmExcept(dirName, keepPrefix) { finally { if (e_2) throw e_2.error; }
var e_3, _a; }
const dir = await external_fs_default().promises.opendir(dirName); const keepPkg = new Set(packages.map((p) => p.name));
try { await rmExcept("./target/debug/build", keepPkg);
for (var dir_3 = save_asyncValues(dir), dir_3_1; dir_3_1 = await dir_3.next(), !dir_3_1.done;) { await rmExcept("./target/debug/.fingerprint", keepPkg);
const dirent = dir_3_1.value; const keepDeps = new Set(packages.flatMap((p) => {
let name = dirent.name; const names = [];
const idx = name.lastIndexOf("-"); for (const n of [p.name, ...p.targets]) {
if (idx !== -1) { const name = n.replace(/-/g, "_");
name = name.slice(0, idx); names.push(name, `lib${name}`);
} }
const fileName = external_path_default().join(dir.path, dirent.name); return names;
const { mtime } = await external_fs_default().promises.stat(fileName); }));
if (!keepPrefix.has(name) || Date.now() - mtime.getTime() > twoWeeks) { await rmExcept("./target/debug/deps", keepDeps);
core.debug(`deleting "${fileName}"`); }
if (dirent.isFile()) { const twoWeeks = 14 * 24 * 3600 * 1000;
await external_fs_default().promises.unlink(fileName); async function rmExcept(dirName, keepPrefix) {
} var e_3, _a;
else if (dirent.isDirectory()) { const dir = await external_fs_default().promises.opendir(dirName);
await io.rmRF(fileName); try {
} for (var dir_3 = save_asyncValues(dir), dir_3_1; dir_3_1 = await dir_3.next(), !dir_3_1.done;) {
} const dirent = dir_3_1.value;
} let name = dirent.name;
} const idx = name.lastIndexOf("-");
catch (e_3_1) { e_3 = { error: e_3_1 }; } if (idx !== -1) {
finally { name = name.slice(0, idx);
try { }
if (dir_3_1 && !dir_3_1.done && (_a = dir_3.return)) await _a.call(dir_3); const fileName = external_path_default().join(dir.path, dirent.name);
} const { mtime } = await external_fs_default().promises.stat(fileName);
finally { if (e_3) throw e_3.error; } if (!keepPrefix.has(name) || Date.now() - mtime.getTime() > twoWeeks) {
} core.debug(`deleting "${fileName}"`);
} if (dirent.isFile()) {
await external_fs_default().promises.unlink(fileName);
}
else if (dirent.isDirectory()) {
await io.rmRF(fileName);
}
}
}
}
catch (e_3_1) { e_3 = { error: e_3_1 }; }
finally {
try {
if (dir_3_1 && !dir_3_1.done && (_a = dir_3.return)) await _a.call(dir_3);
}
finally { if (e_3) throw e_3.error; }
}
}
async function macOsWorkaround() {
try {
// Workaround for https://github.com/actions/cache/issues/403
// Also see https://github.com/rust-lang/cargo/issues/8603
await exec.exec("sudo", ["/usr/sbin/purge"], { silent: true });
}
catch (_a) { }
}
/***/ }), /***/ }),

2
package-lock.json generated
View File

@@ -1,6 +1,6 @@
{ {
"name": "rust-cache", "name": "rust-cache",
"version": "1.0.0", "version": "1.0.2",
"lockfileVersion": 1, "lockfileVersion": 1,
"requires": true, "requires": true,
"dependencies": { "dependencies": {

View File

@@ -1,7 +1,7 @@
{ {
"private": true, "private": true,
"name": "rust-cache", "name": "rust-cache",
"version": "1.0.0", "version": "1.0.2",
"description": "A GitHub Action that implements smart caching for rust/cargo projects", "description": "A GitHub Action that implements smart caching for rust/cargo projects",
"keywords": [ "keywords": [
"actions", "actions",

View File

@@ -14,16 +14,17 @@ export const paths = {
target: "target", target: "target",
}; };
export interface CacheConfig { interface CacheConfig {
name: string;
path: string; path: string;
key: string; key: string;
restoreKeys?: Array<string>; restoreKeys?: Array<string>;
} }
export interface Caches { interface Caches {
index: CacheConfig; index: CacheConfig;
cache: CacheConfig; cache: CacheConfig;
git: CacheConfig; // git: CacheConfig;
target: CacheConfig; target: CacheConfig;
} }
@@ -44,19 +45,38 @@ export async function getCaches(): Promise<Caches> {
if (targetKey) { if (targetKey) {
targetKey = `${targetKey}-`; targetKey = `${targetKey}-`;
} }
const registryIndex = `v0-registry-index`;
const registryCache = `v0-registry-cache`;
const target = `v0-target-${targetKey}${rustKey}`;
return { return {
index: { path: paths.index, key: "registry-index-XXX", restoreKeys: ["registry-index"] }, index: {
cache: { path: paths.cache, key: `registry-cache-${lockHash}`, restoreKeys: ["registry-cache"] }, name: "Registry Index",
git: { path: paths.git, key: "git-db" }, path: paths.index,
key: `${registryIndex}-`,
restoreKeys: [registryIndex],
},
cache: {
name: "Registry Cache",
path: paths.cache,
key: `${registryCache}-${lockHash}`,
restoreKeys: [registryCache],
},
// git: {
// name: "Git Dependencies",
// path: paths.git,
// key: "git-db",
// },
target: { target: {
name: "Target",
path: paths.target, path: paths.target,
key: `target-${targetKey}${rustKey}-${lockHash}`, key: `${target}-${lockHash}`,
restoreKeys: [`target-${targetKey}${rustKey}`], restoreKeys: [target],
}, },
}; };
} }
export async function getRustKey(): Promise<string> { async function getRustKey(): Promise<string> {
const rustc = await getRustVersion(); const rustc = await getRustVersion();
return `${rustc.release}-${rustc.host}-${rustc["commit-hash"]}`; return `${rustc.release}-${rustc.host}-${rustc["commit-hash"]}`;
} }
@@ -67,7 +87,7 @@ interface RustVersion {
"commit-hash": string; "commit-hash": string;
} }
export async function getRustVersion(): Promise<RustVersion> { async function getRustVersion(): Promise<RustVersion> {
const stdout = await getCmdOutput("rustc", ["-vV"]); const stdout = await getCmdOutput("rustc", ["-vV"]);
let splits = stdout let splits = stdout
.split(/[\n\r]+/) .split(/[\n\r]+/)
@@ -109,7 +129,7 @@ export async function getRegistryName() {
return path.basename(path.dirname(first)); return path.basename(path.dirname(first));
} }
export async function getLockfileHash() { async function getLockfileHash() {
const globber = await glob.create("**/Cargo.toml\n**/Cargo.lock", { followSymbolicLinks: false }); const globber = await glob.create("**/Cargo.toml\n**/Cargo.lock", { followSymbolicLinks: false });
const files = await globber.glob(); const files = await globber.glob();
files.sort((a, b) => a.localeCompare(b)); files.sort((a, b) => a.localeCompare(b));

View File

@@ -11,21 +11,27 @@ async function run() {
core.exportVariable("CARGO_INCREMENTAL", 0); core.exportVariable("CARGO_INCREMENTAL", 0);
const caches = await getCaches(); const caches = await getCaches();
for (const [name, { path, key, restoreKeys }] of Object.entries(caches)) { for (const [type, { name, path, key, restoreKeys }] of Object.entries(caches)) {
const start = Date.now();
core.startGroup(`Restoring ${name}`);
core.info(`Restoring to path "${path}".`);
core.info(`Using keys:\n ${[key, ...restoreKeys].join("\n ")}`);
try { try {
core.startGroup(`Restoring "${path}" from "${key}"…`);
const restoreKey = await cache.restoreCache([path], key, restoreKeys); const restoreKey = await cache.restoreCache([path], key, restoreKeys);
if (restoreKey) { if (restoreKey) {
core.info(`Restored "${path}" from cache key "${restoreKey}".`); core.info(`Restored from cache key "${restoreKey}".`);
core.saveState(name, restoreKey); core.saveState(type, restoreKey);
} else { } else {
core.info("No cache found."); core.info("No cache found.");
} }
} catch (e) { } catch (e) {
core.info(`[warning] ${e.message}`); core.info(`[warning] ${e.message}`);
} finally {
core.endGroup();
} }
const duration = Math.round((Date.now() - start) / 1000);
if (duration) {
core.info(`Took ${duration}s.`);
}
core.endGroup();
} }
} catch (e) { } catch (e) {
core.info(`[warning] ${e.message}`); core.info(`[warning] ${e.message}`);

View File

@@ -1,5 +1,6 @@
import * as cache from "@actions/cache"; import * as cache from "@actions/cache";
import * as core from "@actions/core"; import * as core from "@actions/core";
import * as exec from "@actions/exec";
import * as io from "@actions/io"; import * as io from "@actions/io";
import fs from "fs"; import fs from "fs";
import path from "path"; import path from "path";
@@ -7,7 +8,7 @@ import { getCaches, getCmdOutput, getRegistryName, isValidEvent, paths } from ".
async function run() { async function run() {
if (!isValidEvent()) { if (!isValidEvent()) {
//return; return;
} }
try { try {
@@ -15,11 +16,15 @@ async function run() {
const registryName = await getRegistryName(); const registryName = await getRegistryName();
const packages = await getPackages(); const packages = await getPackages();
// TODO: remove this once https://github.com/actions/toolkit/pull/553 lands
await macOsWorkaround();
await pruneTarget(packages); await pruneTarget(packages);
if (registryName) { if (registryName) {
// save the index based on its revision // save the index based on its revision
const indexRef = await getIndexRef(registryName); const indexRef = await getIndexRef(registryName);
caches.index.key = `registry-index-${indexRef}`; caches.index.key += indexRef;
await io.rmRF(path.join(paths.index, registryName, ".cache")); await io.rmRF(path.join(paths.index, registryName, ".cache"));
await pruneRegistryCache(registryName, packages); await pruneRegistryCache(registryName, packages);
@@ -28,21 +33,25 @@ async function run() {
delete (caches as any).cache; delete (caches as any).cache;
} }
for (const [name, { path, key }] of Object.entries(caches)) { for (const [type, { name, path, key }] of Object.entries(caches)) {
if (core.getState(name) === key) { if (core.getState(type) === key) {
core.info(`Cache for "${path}" up-to-date.`); core.info(`${name} up-to-date.`);
continue; continue;
} }
const start = Date.now();
core.startGroup(`Saving ${name}`);
core.info(`Saving path "${path}".`);
core.info(`Using key "${key}".`);
try { try {
core.startGroup(`Saving "${path}" to cache key "${key}"…`); await cache.saveCache([path], key);
if (await cache.saveCache([path], key)) {
core.info(`Saved "${path}" to cache key "${key}".`);
}
} catch (e) { } catch (e) {
core.info(`[warning] ${e.message}`); core.info(`[warning] ${e.message}`);
} finally {
core.endGroup();
} }
const duration = Math.round((Date.now() - start) / 1000);
if (duration) {
core.info(`Took ${duration}s.`);
}
core.endGroup();
} }
} catch (e) { } catch (e) {
core.info(`[warning] ${e.message}`); core.info(`[warning] ${e.message}`);
@@ -59,13 +68,30 @@ async function getIndexRef(registryName: string) {
interface PackageDefinition { interface PackageDefinition {
name: string; name: string;
version: string; version: string;
targets: Array<string>;
} }
type Packages = Array<PackageDefinition>; type Packages = Array<PackageDefinition>;
interface Meta {
packages: Array<{
name: string;
version: string;
manifest_path: string;
targets: Array<{ kind: Array<string>; name: string }>;
}>;
}
async function getPackages(): Promise<Packages> { async function getPackages(): Promise<Packages> {
const meta = JSON.parse(await getCmdOutput("cargo", ["metadata", "--format-version", "1"])); const cwd = process.cwd();
return meta.packages.map(({ name, version }: any) => ({ name, version })); const meta: Meta = JSON.parse(await getCmdOutput("cargo", ["metadata", "--all-features", "--format-version", "1"]));
return meta.packages
.filter((p) => !p.manifest_path.startsWith(cwd))
.map((p) => {
const targets = p.targets.filter((t) => t.kind[0] === "lib").map((t) => t.name);
return { name: p.name, version: p.version, targets };
});
} }
async function pruneRegistryCache(registryName: string, packages: Packages) { async function pruneRegistryCache(registryName: string, packages: Packages) {
@@ -102,8 +128,12 @@ async function pruneTarget(packages: Packages) {
const keepDeps = new Set( const keepDeps = new Set(
packages.flatMap((p) => { packages.flatMap((p) => {
const name = p.name.replace(/-/g, "_"); const names = [];
return [name, `lib${name}`]; for (const n of [p.name, ...p.targets]) {
const name = n.replace(/-/g, "_");
names.push(name, `lib${name}`);
}
return names;
}), }),
); );
await rmExcept("./target/debug/deps", keepDeps); await rmExcept("./target/debug/deps", keepDeps);
@@ -131,3 +161,11 @@ async function rmExcept(dirName: string, keepPrefix: Set<string>) {
} }
} }
} }
async function macOsWorkaround() {
try {
// Workaround for https://github.com/actions/cache/issues/403
// Also see https://github.com/rust-lang/cargo/issues/8603
await exec.exec("sudo", ["/usr/sbin/purge"], { silent: true });
} catch {}
}