Skip to content

Commit

Permalink
*: replace the bazelisk, disk and repo cache with sticky disks
Browse files Browse the repository at this point in the history
This change teaches restoreCache to get and mount a unique stickydisk
per key and mount it at the provided path. We require a stickydisk per
path and so we hash the path and add it to the user configured "cache key".
All other semantics around versioning of the cache and respecting the base
cache key remain the same.

On post, we teach the action to unmount and commit the relevant stickydisks.
For the moment we leave the "external" cache alone. In a follow-up we can move
that to be backed by stickydisks too.

The stickydisk.js file is heavily inspired by the useblacksmith/stickydisk
implementation.
  • Loading branch information
adityamaru committed Dec 14, 2024
1 parent beb23e2 commit a347c1e
Show file tree
Hide file tree
Showing 17 changed files with 128,982 additions and 97,059 deletions.
15 changes: 15 additions & 0 deletions .github/workflows/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,21 @@ concurrency:
cancel-in-progress: true

jobs:
test:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- uses: actions/setup-node@v4
with:
node-version: '20'
- uses: bufbuild/buf-setup-action@v1
- run: |
npm config set @buf:registry https://buf.build/gen/npm/v1/
npm config set //buf.build/gen/npm/v1/:_authToken ${{ secrets.BUF_TOKEN }}
npm install @buf/blacksmith_vm-agent.connectrpc_es@latest
- run: npm ci
- run: npm test

setup-bazel:
runs-on: ${{ matrix.os }}-latest
strategy:
Expand Down
339 changes: 339 additions & 0 deletions __tests__/restore-cache.test.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,339 @@
const core = require('@actions/core');
const { mountStickyDisk } = require('../stickydisk');

// Mock the stickydisk module
jest.mock('../stickydisk', () => ({
mountStickyDisk: jest.fn()
}));

// Mock YAML parser
jest.mock('yaml', () => ({
parse: jest.fn().mockImplementation((input) => {
if (typeof input === 'object') {
return input;
}
return {};
})
}));

// Mock github context
jest.mock('@actions/github', () => ({
context: {
workflow: 'test-workflow',
job: 'test-job',
repo: {
owner: 'test-owner',
repo: 'test-repo'
},
sha: '1234567890abcdef',
ref: 'refs/heads/main'
}
}));

// Mock core with getInput
jest.mock('@actions/core', () => ({
debug: jest.fn(),
info: jest.fn(),
warning: jest.fn(),
startGroup: jest.fn(),
endGroup: jest.fn(),
getState: jest.fn((name) => {
const mockState = {
'sticky-disk-mounts': '{}',
'google-credentials-path': '',
};
return mockState[name] || '';
}),
saveState: jest.fn(),
toPosixPath: jest.fn(path => path.replace(/\\/g, '/')),
getInput: jest.fn((name) => {
const mockInputs = {
'bazelisk-version': '1.18.0',
'cache-version': 'v1',
'external-cache': '{}',
'google-credentials': '',
// Add other inputs as needed
};
return mockInputs[name] || '';
}),
getMultilineInput: jest.fn((name) => {
const mockInputs = {
'bazelrc': [],
};
return mockInputs[name] || [];
}),
getBooleanInput: jest.fn((name) => {
const mockInputs = {
'bazelisk-cache': true,
};
return mockInputs[name] || false;
}),
setFailed: jest.fn(),
}));

// Mock glob.hashFiles
jest.mock('@actions/glob', () => ({
hashFiles: jest.fn().mockResolvedValue('testhash123')
}));


// Import the module under test after mocking dependencies
const { loadStickyDisk, loadExternalStickyDisks } = require('../index');

// Add helper to set state
function setState(state) {
core.getState.mockImplementation((name) => {
const defaultState = {
'sticky-disk-mounts': '{}',
'google-credentials-path': '',
};
return state[name] || defaultState[name] || '';
});
}

function setupTestConfig(inputs = {}) {
// Reset all mocks
jest.clearAllMocks();

// Set default inputs
core.getInput.mockImplementation((name) => {
const defaultInputs = {
'bazelisk-version': '1.18.0',
'cache-version': 'v1',
'external-cache': '{}',
'google-credentials': '',
...inputs // Override defaults with test-specific inputs
};
return defaultInputs[name] || '';
});

// Reset multiline inputs
core.getMultilineInput.mockImplementation((name) => {
const defaultInputs = {
'bazelrc': [],
...(inputs.multiline || {}) // Allow overriding multiline inputs
};
return defaultInputs[name] || [];
});

// Reset state
setState(inputs.state || {});

// Re-import config to get fresh instance with new inputs
let freshConfig;
jest.isolateModules(() => {
freshConfig = require('../config');
});
return freshConfig;
}

describe('loadStickyDisk', () => {
it('should mount sticky disk for repository cache when enabled', async () => {
const testConfig = setupTestConfig({
'repository-cache': true,
});

const repositoryCache = testConfig.repositoryCache;

mountStickyDisk.mockResolvedValue({
device: '/dev/sda1',
exposeId: 'test-expose-id'
});

await loadStickyDisk(repositoryCache);

// Verify mountStickyDisk was called with correct parameters.
expect(mountStickyDisk).toHaveBeenCalledWith(
expect.stringMatching(/repository-testhash123-[a-f0-9]{8}/),
expect.stringMatching(/.*\/\.cache\/bazel-repo/), // Match actual path from config
expect.any(AbortSignal),
expect.any(AbortController)
);

// Verify state was saved correctly.
expect(core.saveState).toHaveBeenCalledWith(
'sticky-disk-mounts',
expect.stringContaining('bazel-repo')
);

// Verify the saved state contains the expected mount info
const savedState = JSON.parse(core.saveState.mock.calls[0][1]);
const mountPath = Object.keys(savedState)[0];
expect(savedState[mountPath]).toEqual({
device: '/dev/sda1',
exposeId: 'test-expose-id',
stickyDiskKey: expect.stringMatching(/repository-testhash123-[a-f0-9]{8}/)
});

// Verify bazelrc was updated with repository cache path
expect(testConfig.bazelrc).toContain(`build --repository_cache=${testConfig.repositoryCache.paths[0]}`);
});

it('should mount sticky disk for disk cache when enabled', async () => {
const testConfig = setupTestConfig({
'disk-cache': true,
});

const diskCache = testConfig.diskCache;

mountStickyDisk.mockResolvedValue({
device: '/dev/sda1',
exposeId: 'test-expose-id'
});

await loadStickyDisk(diskCache);

// Verify mountStickyDisk was called with correct parameters.
expect(mountStickyDisk).toHaveBeenCalledWith(
expect.stringMatching(/disk-true-testhash123-[a-f0-9]{8}/),
expect.stringMatching(/.*\/\.cache\/bazel-disk/), // Match actual path from config
expect.any(AbortSignal),
expect.any(AbortController)
);

// Verify state was saved correctly.
expect(core.saveState).toHaveBeenCalledWith(
'sticky-disk-mounts',
expect.stringContaining('bazel-disk')
);

// Verify the saved state contains the expected mount info
const savedState = JSON.parse(core.saveState.mock.calls[0][1]);
const mountPath = Object.keys(savedState)[0];
expect(savedState[mountPath]).toEqual({
device: '/dev/sda1',
exposeId: 'test-expose-id',
stickyDiskKey: expect.stringMatching(/disk-true-testhash123-[a-f0-9]{8}/)
});

// Verify bazelrc was updated with disk cache path
expect(testConfig.bazelrc).toContain(`build --disk_cache=${testConfig.diskCache.paths[0]}`);
});

it('should mount sticky disk for bazelisk cache when enabled', async () => {
const testConfig = setupTestConfig({
'bazelisk-cache': true,
});

const bazeliskCache = testConfig.bazeliskCache;

mountStickyDisk.mockResolvedValue({
device: '/dev/sda1',
exposeId: 'test-expose-id'
});

await loadStickyDisk(bazeliskCache);

// Verify mountStickyDisk was called with correct parameters
expect(mountStickyDisk).toHaveBeenCalledWith(
expect.stringMatching(/bazelisk-testhash123-[a-f0-9]{8}/),
expect.stringMatching(/.*\/bazelisk/), // Match bazelisk cache path
expect.any(AbortSignal),
expect.any(AbortController)
);

// Verify state was saved correctly
expect(core.saveState).toHaveBeenCalledWith(
'sticky-disk-mounts',
expect.stringContaining('bazelisk')
);

// Verify the saved state contains the expected mount info
const savedState = JSON.parse(core.saveState.mock.calls[0][1]);
const mountPath = Object.keys(savedState)[0];
expect(savedState[mountPath]).toEqual({
device: '/dev/sda1',
exposeId: 'test-expose-id',
stickyDiskKey: expect.stringMatching(/bazelisk-testhash123-[a-f0-9]{8}/)
});
});

it('should not mount sticky disk for bazelisk cache when disabled', async () => {
// Override getBooleanInput to return false for bazelisk-cache
core.getBooleanInput.mockImplementation((name) => {
const mockInputs = {
'bazelisk-cache': false,
};
return mockInputs[name] || false;
});

const testConfig = setupTestConfig({
'bazelisk-cache': false,
});

const bazeliskCache = testConfig.bazeliskCache;

await loadStickyDisk(bazeliskCache);

// Verify mountStickyDisk was not called
expect(mountStickyDisk).not.toHaveBeenCalled();

// Verify no state was saved
expect(core.saveState).not.toHaveBeenCalledWith(
'sticky-disk-mounts',
expect.any(String)
);
});

it('should mount sticky disks for all caches when enabled', async () => {
// Override getBooleanInput to return false for bazelisk-cache
core.getBooleanInput.mockImplementation((name) => {
const mockInputs = {
'bazelisk-cache': true,
};
return mockInputs[name] || false;
});
const testConfig = setupTestConfig({
'disk-cache': 'true',
'repository-cache': 'true',
'bazelisk-cache': true
});

// Load all caches
await loadStickyDisk(testConfig.diskCache);
await loadStickyDisk(testConfig.repositoryCache);
await loadStickyDisk(testConfig.bazeliskCache);

// Verify mountStickyDisk was called 3 times
expect(mountStickyDisk).toHaveBeenCalledTimes(3);

// Verify calls for each cache type
expect(mountStickyDisk).toHaveBeenCalledWith(
expect.stringMatching(/disk-testhash123-[a-f0-9]{8}/),
expect.stringMatching(/.*\/_bazel-disk|.*\/\.cache\/bazel-disk/),
expect.any(AbortSignal),
expect.any(AbortController)
);

expect(mountStickyDisk).toHaveBeenCalledWith(
expect.stringMatching(/repository-testhash123-[a-f0-9]{8}/),
expect.stringMatching(/.*\/_bazel-repo|.*\/\.cache\/bazel-repo/),
expect.any(AbortSignal),
expect.any(AbortController)
);

expect(mountStickyDisk).toHaveBeenCalledWith(
expect.stringMatching(/bazelisk-testhash123-[a-f0-9]{8}/),
expect.stringMatching(/.*\/bazelisk/),
expect.any(AbortSignal),
expect.any(AbortController)
);

// Verify state was saved with all mounts
// Verify state was saved 3 times with expected mount info
expect(core.saveState.mock.calls).toEqual([
[
'sticky-disk-mounts',
expect.stringMatching(/{".*":{.*"device":"\/dev\/sda1","exposeId":"test-expose-id","stickyDiskKey":".*disk-testhash123-[a-f0-9]{8}"}}/)
],
[
'sticky-disk-mounts',
expect.stringMatching(/{".*":{.*"device":"\/dev\/sda1","exposeId":"test-expose-id","stickyDiskKey":".*repository-testhash123-[a-f0-9]{8}"}}/)
],
[
'sticky-disk-mounts',
expect.stringMatching(/{".*":{.*"device":"\/dev\/sda1","exposeId":"test-expose-id","stickyDiskKey":".*bazelisk-testhash123-[a-f0-9]{8}"}}/)
]
]);
});
});
2 changes: 1 addition & 1 deletion config.js
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,7 @@ const diskCacheEnabled = diskCacheConfig !== 'false'
let diskCacheName = 'disk'
if (diskCacheEnabled) {
bazelrc.push(`build --disk_cache=${bazelDisk}`)
if (diskCacheName !== 'true') {
if (diskCacheConfig !== 'true') {
diskCacheName = `${diskCacheName}-${diskCacheConfig}`
}
}
Expand Down
Loading

0 comments on commit a347c1e

Please sign in to comment.