Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

use restore keys to restore cache from partial key matches + cache manifest install directory #94

Closed
wants to merge 2 commits into from
Closed

use restore keys to restore cache from partial key matches + cache manifest install directory #94

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
8d2d8e3
use restore keys to restore cache from partial key matches
Be-ing Aug 6, 2021
022f6ed
cache vcpkg's binary cache instead of install or vcpkg_install
Be-ing Aug 6, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 3 additions & 5 deletions __tests__/basic.test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -128,11 +128,9 @@ test('run-vcpkg: cache must not be restored/saved when "doNotCache" is true scen
});

test('isExactKeyMatch tests', () => {
const hash = vcpkgutils.Utils.hashCode("hashCode");
Be-ing marked this conversation as resolved.
Show resolved Hide resolved
const hash2 = vcpkgutils.Utils.hashCode("hashCode2");
expect(vcpkgutils.Utils.isExactKeyMatch(hash, hash2)).toBeFalsy();
expect(vcpkgutils.Utils.isExactKeyMatch(hash2, hash)).toBeFalsy();
expect(vcpkgutils.Utils.isExactKeyMatch(hash, hash)).toBeTruthy();
expect(vcpkgutils.Utils.isExactKeyMatch("1", "2")).toBeFalsy();
expect(vcpkgutils.Utils.isExactKeyMatch("2", "1")).toBeFalsy();
expect(vcpkgutils.Utils.isExactKeyMatch("1", "1")).toBeTruthy();
});

test('getVcpkgCommitId tests', async () => {
Expand Down
88 changes: 46 additions & 42 deletions dist/index.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -109,14 +109,14 @@ class VcpkgAction {
}
run() {
return __awaiter(this, void 0, void 0, function* () {
const vcpkgCacheComputedKey = yield vcpkgutil.Utils.computeCacheKey(this.appendedCacheKey);
const restoreKeys = yield vcpkgutil.Utils.computeCacheKey(this.appendedCacheKey);
const vcpkgCacheComputedKey = restoreKeys[0];
if (!vcpkgCacheComputedKey) {
core.error("Computation for the cache key failed!");
}
else {
core.saveState(exports.VCPKG_CACHE_COMPUTED_KEY, vcpkgCacheComputedKey);
core.info(`Cache's key = '${vcpkgCacheComputedKey}'.`);
yield this.baseUtilLib.wrapOp('Restore vcpkg and its artifacts from cache', () => this.restoreCache(vcpkgCacheComputedKey));
yield this.baseUtilLib.wrapOp('Restore vcpkg and its artifacts from cache', () => this.restoreCache(restoreKeys));
const runner = new runvcpkglib.VcpkgRunner(this.baseUtilLib.baseLib);
yield runner.run();
if (this.isSetupOnly) {
Expand All @@ -131,27 +131,30 @@ class VcpkgAction {
}
saveCache(key) {
return __awaiter(this, void 0, void 0, function* () {
yield vcpkgutil.Utils.saveCache(this.doNotCache, key, this.hitCacheKey, vcpkgutil.Utils.getAllCachedPaths(this.baseUtilLib.baseLib, this.vcpkgRootDir));
yield vcpkgutil.Utils.saveCache(this.doNotCache, key, this.hitCacheKey, vcpkgutil.Utils.getAllCachedPaths(this.baseUtilLib, this.vcpkgRootDir));
});
}
restoreCache(key) {
restoreCache(restoreKeys) {
return __awaiter(this, void 0, void 0, function* () {
try {
if (this.doNotCache) {
core.info(`Caching is disabled (${exports.doNotCacheInput}=true)`);
}
else {
const pathsToCache = vcpkgutil.Utils.getAllCachedPaths(this.baseUtilLib.baseLib, this.vcpkgRootDir);
core.info(`Cache's key = '${key}', paths = '${pathsToCache}'`);
const pathsToCache = vcpkgutil.Utils.getAllCachedPaths(this.baseUtilLib, this.vcpkgRootDir);
const primaryKey = restoreKeys.shift();
core.info(`Cache key: '${primaryKey}'`);
core.info(`Cache restore keys: '${restoreKeys}'`);
core.info(`Cached paths: '${pathsToCache}'`);
core.info(`Running restore-cache...`);
let cacheHitId;
try {
cacheHitId = yield cache.restoreCache(pathsToCache, key);
cacheHitId = yield cache.restoreCache(pathsToCache, primaryKey, restoreKeys);
}
catch (err) {
try {
core.warning(`restoreCache() failed once: '${err === null || err === void 0 ? void 0 : err.toString()}' , retrying...`);
cacheHitId = yield cache.restoreCache(pathsToCache, key);
cacheHitId = yield cache.restoreCache(pathsToCache, primaryKey, restoreKeys);
}
catch (err) {
core.warning(`restoreCache() failed again: '${err === null || err === void 0 ? void 0 : err.toString()}'.`);
Expand Down Expand Up @@ -220,21 +223,6 @@ class Utils {
}
}
}
/**
* Compute an unique string given some text.
* @param {string} text The text to computer an hash for.
* @returns {string} The unique hash of 'text'.
*/
static hashCode(text) {
let hash = 42;
if (text.length != 0) {
for (let i = 0; i < text.length; i++) {
const char = text.charCodeAt(i);
hash = ((hash << 5) + hash) ^ char;
}
}
return hash.toString();
}
static isExactKeyMatch(key, cacheKey) {
if (cacheKey)
return cacheKey.localeCompare(key, undefined, { sensitivity: "accent" }) === 0;
Expand Down Expand Up @@ -287,38 +275,44 @@ class Utils {
static computeCacheKey(appendedCacheKey) {
return __awaiter(this, void 0, void 0, function* () {
let key = "";
const inputVcpkgPath = core.getInput(runvcpkglib.vcpkgDirectory);
const restoreKeys = [];
const actionLib = new actionlib.ActionLib();
const baseUtil = new baseutillib.BaseUtilLib(actionLib);
key += "runnerOS=" + process.env.ImageOS ? process.env.ImageOS : 0;
// Add the triplet only if it is provided.
const triplet = core.getInput(runvcpkglib.vcpkgTriplet);
if (triplet) {
key += "-triplet=" + triplet;
}
restoreKeys.push(key);
const inputVcpkgPath = core.getInput(runvcpkglib.vcpkgDirectory);
const [commitId, isSubmodule] = yield Utils.getVcpkgCommitId(baseUtil, inputVcpkgPath);
const userProvidedCommitId = core.getInput(runvcpkglib.vcpkgCommitId);
if (commitId) {
core.info(`vcpkg identified at commitId='${commitId}', adding it to the cache's key.`);
if (isSubmodule) {
key += `submodGitId=${commitId}`;
key += `_vcpkgGitCommit=${commitId}`;
core.info(`Adding vcpkg submodule commit id '${commitId}' to cache key`);
}
else {
key += "localGitId=" + Utils.hashCode(userProvidedCommitId);
key += `_vcpkgGitCommit=${userProvidedCommitId}`;
core.info(`Adding user provided vcpkg commit id ${userProvidedCommitId} to cache key`);
}
restoreKeys.push(key);
}
else if (userProvidedCommitId) {
core.info(`Using user provided vcpkg's Git commit id='${userProvidedCommitId}', adding it to the cache's key.`);
key += "localGitId=" + Utils.hashCode(userProvidedCommitId);
key += `_vcpkgGitCommit=${userProvidedCommitId}`;
core.info(`Adding user provided vcpkg commit id ${userProvidedCommitId} to cache key`);
restoreKeys.push(key);
}
else {
core.info(`No vcpkg's commit id was provided, does not contribute to the cache's key.`);
core.info(`No vcpkg commit id was provided, does not contribute to the cache's key.`);
}
Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I would like to put the hash of the vcpkg.json file before the commit hash (#95) in the cache key. Once a project is setup I would think the vcpkg.json file would change infrequently. Updating the packages would simply be a matter of changing the commit hash/updating the submodule. When doing so, the cache should hit for the last cache with the same vcpkg.json to avoid having to rebuild every package. But this needs to wait for the hashFiles function to be published in the @actions/glob NPM package (actions/toolkit#830).

Copy link
Owner

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

is the hashFiles function really necessary? Or any hash function should be a good fit?

Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I suppose another hash function could do 🤷

Copy link
Owner

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I still see that your point of using eactly the same hash function has its benefits: it would not invalidate existing keys. On the other hands, this PR already invalidate existing keys as the key changed its format, so no concern here. Any hash function would be a good fit.

key += "-args=" + Utils.hashCode(core.getInput(runvcpkglib.vcpkgArguments));
key += "-os=" + Utils.hashCode(process.env.ImageOS ? process.env.ImageOS : process.platform);
if (process.env.ImageVersion) {
key += "-imageVer=" + Utils.hashCode(process.env.ImageVersion);
if (appendedCacheKey) {
key += `_appendedKey=${appendedCacheKey}`;
restoreKeys.push(key);
}
key += "-appendedKey=" + Utils.hashCode(appendedCacheKey);
// Add the triplet only if it is provided.
const triplet = core.getInput(runvcpkglib.vcpkgTriplet);
if (triplet)
key += "-triplet=" + Utils.hashCode(triplet);
return key;
restoreKeys.reverse();
return restoreKeys;
});
}
static saveCache(doNotCache, vcpkgCacheComputedKey, hitCacheKey, cachedPaths) {
Expand Down Expand Up @@ -370,8 +364,18 @@ class Utils {
core.saveState(Utils.VCPKG_ADDITIONAL_CACHED_PATHS_KEY, paths);
core.exportVariable(Utils.VCPKG_ADDITIONAL_CACHED_PATHS_KEY, paths);
}
static getAllCachedPaths(baselib, vcpkgRootDir) {
static getAllCachedPaths(baseUtils, vcpkgRootDir) {
let paths = runvcpkglib.getOrdinaryCachedPaths(vcpkgRootDir);
// Use vcpkg's binary cache only; do not double cache the `installed` directory because
// that would inflate the cache size unnecessarily.
const installed_path = path.resolve(vcpkgRootDir + "/installed");
paths.push("!" + installed_path);
if (baseUtils.isWin32()) {
paths.push(process.env.LOCALAPPDATA + "\\vcpkg");
}
else {
paths.push(process.env.HOME + "/.cache/vcpkg");
}
let additionalCachedPaths = core.getState(Utils.VCPKG_ADDITIONAL_CACHED_PATHS_KEY);
if (!additionalCachedPaths) {
additionalCachedPaths = process.env[Utils.VCPKG_ADDITIONAL_CACHED_PATHS_KEY];
Expand Down
90 changes: 47 additions & 43 deletions dist/post/index.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -43,7 +43,7 @@ function main() {
const cacheHit = core.getState(vcpkgaction.VCPKG_CACHE_HIT_KEY);
const computedCacheKey = core.getState(vcpkgaction.VCPKG_CACHE_COMPUTED_KEY);
const vcpkgRoot = core.getState(vcpkgaction.VCPKG_ROOT_KEY);
const cachedPaths = vcpkgutil.Utils.getAllCachedPaths(actionLib, vcpkgRoot);
const cachedPaths = vcpkgutil.Utils.getAllCachedPaths(baseUtil, vcpkgRoot);
yield vcpkgutil.Utils.saveCache(doNotCache, computedCacheKey, cacheHit, cachedPaths);
}
}));
Expand Down Expand Up @@ -121,14 +121,14 @@ class VcpkgAction {
}
run() {
return __awaiter(this, void 0, void 0, function* () {
const vcpkgCacheComputedKey = yield vcpkgutil.Utils.computeCacheKey(this.appendedCacheKey);
const restoreKeys = yield vcpkgutil.Utils.computeCacheKey(this.appendedCacheKey);
const vcpkgCacheComputedKey = restoreKeys[0];
if (!vcpkgCacheComputedKey) {
core.error("Computation for the cache key failed!");
}
else {
core.saveState(exports.VCPKG_CACHE_COMPUTED_KEY, vcpkgCacheComputedKey);
core.info(`Cache's key = '${vcpkgCacheComputedKey}'.`);
yield this.baseUtilLib.wrapOp('Restore vcpkg and its artifacts from cache', () => this.restoreCache(vcpkgCacheComputedKey));
yield this.baseUtilLib.wrapOp('Restore vcpkg and its artifacts from cache', () => this.restoreCache(restoreKeys));
const runner = new runvcpkglib.VcpkgRunner(this.baseUtilLib.baseLib);
yield runner.run();
if (this.isSetupOnly) {
Expand All @@ -143,27 +143,30 @@ class VcpkgAction {
}
saveCache(key) {
return __awaiter(this, void 0, void 0, function* () {
yield vcpkgutil.Utils.saveCache(this.doNotCache, key, this.hitCacheKey, vcpkgutil.Utils.getAllCachedPaths(this.baseUtilLib.baseLib, this.vcpkgRootDir));
yield vcpkgutil.Utils.saveCache(this.doNotCache, key, this.hitCacheKey, vcpkgutil.Utils.getAllCachedPaths(this.baseUtilLib, this.vcpkgRootDir));
});
}
restoreCache(key) {
restoreCache(restoreKeys) {
return __awaiter(this, void 0, void 0, function* () {
try {
if (this.doNotCache) {
core.info(`Caching is disabled (${exports.doNotCacheInput}=true)`);
}
else {
const pathsToCache = vcpkgutil.Utils.getAllCachedPaths(this.baseUtilLib.baseLib, this.vcpkgRootDir);
core.info(`Cache's key = '${key}', paths = '${pathsToCache}'`);
const pathsToCache = vcpkgutil.Utils.getAllCachedPaths(this.baseUtilLib, this.vcpkgRootDir);
const primaryKey = restoreKeys.shift();
core.info(`Cache key: '${primaryKey}'`);
core.info(`Cache restore keys: '${restoreKeys}'`);
core.info(`Cached paths: '${pathsToCache}'`);
core.info(`Running restore-cache...`);
let cacheHitId;
try {
cacheHitId = yield cache.restoreCache(pathsToCache, key);
cacheHitId = yield cache.restoreCache(pathsToCache, primaryKey, restoreKeys);
}
catch (err) {
try {
core.warning(`restoreCache() failed once: '${err === null || err === void 0 ? void 0 : err.toString()}' , retrying...`);
cacheHitId = yield cache.restoreCache(pathsToCache, key);
cacheHitId = yield cache.restoreCache(pathsToCache, primaryKey, restoreKeys);
}
catch (err) {
core.warning(`restoreCache() failed again: '${err === null || err === void 0 ? void 0 : err.toString()}'.`);
Expand Down Expand Up @@ -232,21 +235,6 @@ class Utils {
}
}
}
/**
* Compute an unique string given some text.
* @param {string} text The text to computer an hash for.
* @returns {string} The unique hash of 'text'.
*/
static hashCode(text) {
let hash = 42;
if (text.length != 0) {
for (let i = 0; i < text.length; i++) {
const char = text.charCodeAt(i);
hash = ((hash << 5) + hash) ^ char;
}
}
return hash.toString();
}
static isExactKeyMatch(key, cacheKey) {
if (cacheKey)
return cacheKey.localeCompare(key, undefined, { sensitivity: "accent" }) === 0;
Expand Down Expand Up @@ -299,38 +287,44 @@ class Utils {
static computeCacheKey(appendedCacheKey) {
return __awaiter(this, void 0, void 0, function* () {
let key = "";
const inputVcpkgPath = core.getInput(runvcpkglib.vcpkgDirectory);
const restoreKeys = [];
const actionLib = new actionlib.ActionLib();
const baseUtil = new baseutillib.BaseUtilLib(actionLib);
key += "runnerOS=" + process.env.ImageOS ? process.env.ImageOS : 0;
// Add the triplet only if it is provided.
const triplet = core.getInput(runvcpkglib.vcpkgTriplet);
if (triplet) {
key += "-triplet=" + triplet;
}
restoreKeys.push(key);
const inputVcpkgPath = core.getInput(runvcpkglib.vcpkgDirectory);
const [commitId, isSubmodule] = yield Utils.getVcpkgCommitId(baseUtil, inputVcpkgPath);
const userProvidedCommitId = core.getInput(runvcpkglib.vcpkgCommitId);
if (commitId) {
core.info(`vcpkg identified at commitId='${commitId}', adding it to the cache's key.`);
if (isSubmodule) {
key += `submodGitId=${commitId}`;
key += `_vcpkgGitCommit=${commitId}`;
core.info(`Adding vcpkg submodule commit id '${commitId}' to cache key`);
}
else {
key += "localGitId=" + Utils.hashCode(userProvidedCommitId);
key += `_vcpkgGitCommit=${userProvidedCommitId}`;
core.info(`Adding user provided vcpkg commit id ${userProvidedCommitId} to cache key`);
}
restoreKeys.push(key);
}
else if (userProvidedCommitId) {
core.info(`Using user provided vcpkg's Git commit id='${userProvidedCommitId}', adding it to the cache's key.`);
key += "localGitId=" + Utils.hashCode(userProvidedCommitId);
key += `_vcpkgGitCommit=${userProvidedCommitId}`;
core.info(`Adding user provided vcpkg commit id ${userProvidedCommitId} to cache key`);
restoreKeys.push(key);
}
else {
core.info(`No vcpkg's commit id was provided, does not contribute to the cache's key.`);
core.info(`No vcpkg commit id was provided, does not contribute to the cache's key.`);
}
key += "-args=" + Utils.hashCode(core.getInput(runvcpkglib.vcpkgArguments));
key += "-os=" + Utils.hashCode(process.env.ImageOS ? process.env.ImageOS : process.platform);
if (process.env.ImageVersion) {
key += "-imageVer=" + Utils.hashCode(process.env.ImageVersion);
if (appendedCacheKey) {
key += `_appendedKey=${appendedCacheKey}`;
restoreKeys.push(key);
}
key += "-appendedKey=" + Utils.hashCode(appendedCacheKey);
// Add the triplet only if it is provided.
const triplet = core.getInput(runvcpkglib.vcpkgTriplet);
if (triplet)
key += "-triplet=" + Utils.hashCode(triplet);
return key;
restoreKeys.reverse();
return restoreKeys;
});
}
static saveCache(doNotCache, vcpkgCacheComputedKey, hitCacheKey, cachedPaths) {
Expand Down Expand Up @@ -382,8 +376,18 @@ class Utils {
core.saveState(Utils.VCPKG_ADDITIONAL_CACHED_PATHS_KEY, paths);
core.exportVariable(Utils.VCPKG_ADDITIONAL_CACHED_PATHS_KEY, paths);
}
static getAllCachedPaths(baselib, vcpkgRootDir) {
static getAllCachedPaths(baseUtils, vcpkgRootDir) {
let paths = runvcpkglib.getOrdinaryCachedPaths(vcpkgRootDir);
// Use vcpkg's binary cache only; do not double cache the `installed` directory because
// that would inflate the cache size unnecessarily.
const installed_path = path.resolve(vcpkgRootDir + "/installed");
paths.push("!" + installed_path);
if (baseUtils.isWin32()) {
paths.push(process.env.LOCALAPPDATA + "\\vcpkg");
}
else {
paths.push(process.env.HOME + "/.cache/vcpkg");
}
let additionalCachedPaths = core.getState(Utils.VCPKG_ADDITIONAL_CACHED_PATHS_KEY);
if (!additionalCachedPaths) {
additionalCachedPaths = process.env[Utils.VCPKG_ADDITIONAL_CACHED_PATHS_KEY];
Expand Down
Loading