Skip to content

Commit

Permalink
Merge branch 'improvement/CLDSRV-587' into q/8.8
Browse files Browse the repository at this point in the history
  • Loading branch information
@bert-e
bert-e committed Dec 2, 2024
2 parents aff404c + 868f9a2 commit f527769
Show file tree
Hide file tree
Showing 3 changed files with 173 additions and 0 deletions.
101 changes: 101 additions & 0 deletions tests/quota/awsNodeSdk.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -607,6 +607,60 @@ function multiObjectDelete(bucket, keys, size, callback) {
], done);
});

it('should allow writes after deleting data with quotas below the current number of inflights', done => {
const bucket = 'quota-test-bucket8';
const key = 'quota-test-object';
const size = 400;
if (!s3Config.isQuotaInflightEnabled()) {
return done();
}
return async.series([
next => createBucket(bucket, false, next),
// Set the quota to 10 * size (4000)
next => sendRequest(putQuotaVerb, '127.0.0.1:8000', `/${bucket}/?quota=true`,
JSON.stringify({ quota: 10 * size }), config).then(() => next()).catch(err => next(err)),
// Simulate previous operations since last metrics update (4000 bytes)
next => putObject(bucket, `${key}1`, 5 * size, err => {
assert.ifError(err);
return next();
}),
next => putObject(bucket, `${key}2`, 5 * size, err => {
assert.ifError(err);
return next();
}),
next => wait(inflightFlushFrequencyMS * 2, next),
// After metrics update, set the inflights to 0 (simulate end of metrics update)
next => {
scuba.setInflightAsCapacity(bucket);
return next();
},
// Here we have 0 inflight but the stored bytes are 4000 (equal to the quota)
// Should reject new write with QuotaExceeded (4000 + 400)
next => putObject(bucket, `${key}3`, size, err => {
assert.strictEqual(err.code, 'QuotaExceeded');
return next();
}),
next => wait(inflightFlushFrequencyMS * 2, next),
// Should still have 0 as inflight
next => {
assert.strictEqual(scuba.getInflightsForBucket(bucket), 0);
return next();
},
next => wait(inflightFlushFrequencyMS * 2, next),
// Now delete one object (2000 bytes), it should let us write again
next => deleteObject(bucket, `${key}1`, size, next),
next => wait(inflightFlushFrequencyMS * 2, next),
next => putObject(bucket, `${key}4`, 5 * size, err => {
assert.ifError(err);
return next();
}),
// Cleanup
next => deleteObject(bucket, `${key}2`, size, next),
next => deleteObject(bucket, `${key}4`, size, next),
next => deleteBucket(bucket, next),
], done);
});

it('should not increase the inflights when the object is being rewritten with a smaller object', done => {
const bucket = 'quota-test-bucket9';
const key = 'quota-test-object';
Expand Down Expand Up @@ -668,6 +722,53 @@ function multiObjectDelete(bucket, keys, size, callback) {
], done);
});

it('should allow writes after multi-deleting data with quotas below the current number of inflights', done => {
const bucket = 'quota-test-bucket10';
const key = 'quota-test-object';
const size = 400;
if (!s3Config.isQuotaInflightEnabled()) {
return done();
}
return async.series([
next => createBucket(bucket, false, next),
next => sendRequest(putQuotaVerb, '127.0.0.1:8000', `/${bucket}/?quota=true`,
JSON.stringify({ quota: size * 10 }), config).then(() => next()).catch(err => next(err)),
next => putObject(bucket, `${key}1`, size * 5, err => {
assert.ifError(err);
return next();
}),
next => putObject(bucket, `${key}2`, size * 5, err => {
assert.ifError(err);
return next();
}),
next => wait(inflightFlushFrequencyMS * 2, next),
next => {
scuba.setInflightAsCapacity(bucket);
return next();
},
next => putObject(bucket, `${key}3`, size, err => {
assert.strictEqual(err.code, 'QuotaExceeded');
return next();
}),
next => wait(inflightFlushFrequencyMS * 2, next),
next => {
assert.strictEqual(scuba.getInflightsForBucket(bucket), 0);
return next();
},
next => multiObjectDelete(bucket, [`${key}1`, `${key}2`], size * 10, err => {
assert.ifError(err);
return next();
}),
next => wait(inflightFlushFrequencyMS * 2, next),
next => putObject(bucket, `${key}4`, size * 5, err => {
assert.ifError(err);
return next();
}),
next => deleteObject(bucket, `${key}4`, size * 5, next),
next => deleteBucket(bucket, next),
], done);
});

it('should not update the inflights if the API errored after evaluating quotas (deletion)', done => {
const bucket = 'quota-test-bucket11';
const key = 'quota-test-object';
Expand Down
55 changes: 55 additions & 0 deletions tests/unit/api/apiUtils/quotas/quotaUtils.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -140,6 +140,32 @@ describe('validateQuotas (buckets)', () => {
});
});

it('should decrease the inflights by deleting data, and go below 0 to unblock operations', done => {
const result1 = {
bytesTotal: 150,
};
const result2 = {
bytesTotal: 120,
};
QuotaService._getLatestMetricsCallback.yields(null, result1);
QuotaService._getLatestMetricsCallback.onCall(1).yields(null, result2);

validateQuotas(request, mockBucket, {}, ['objectDelete'], 'objectDelete', -5000, false, mockLog, err => {
assert.ifError(err);
assert.strictEqual(QuotaService._getLatestMetricsCallback.calledOnce, true);
assert.strictEqual(QuotaService._getLatestMetricsCallback.calledWith(
'bucket',
'bucketName_1640995200000',
null,
{
action: 'objectDelete',
inflight: -5000,
}
), true);
done();
});
});

it('should return null if quota is not exceeded', done => {
const result1 = {
bytesTotal: 80,
Expand Down Expand Up @@ -364,6 +390,35 @@ describe('validateQuotas (with accounts)', () => {
});
});

it('should decrease the inflights by deleting data, and go below 0 to unblock operations', done => {
const result1 = {
bytesTotal: 150,
};
const result2 = {
bytesTotal: 120,
};
QuotaService._getLatestMetricsCallback.yields(null, result1);
QuotaService._getLatestMetricsCallback.onCall(1).yields(null, result2);

validateQuotas(request, mockBucketNoQuota, {
account: 'test_1',
quota: 1000,
}, ['objectDelete'], 'objectDelete', -5000, false, mockLog, err => {
assert.ifError(err);
assert.strictEqual(QuotaService._getLatestMetricsCallback.callCount, 1);
assert.strictEqual(QuotaService._getLatestMetricsCallback.calledWith(
'account',
'test_1',
null,
{
action: 'objectDelete',
inflight: -5000,
}
), true);
done();
});
});

it('should return null if quota is not exceeded', done => {
const result1 = {
bytesTotal: 80,
Expand Down
17 changes: 17 additions & 0 deletions tests/utilities/mock/Scuba.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -62,6 +62,8 @@ class Scuba {
if (timeout && this.supportsInflight) {
setTimeout(() => {
if (this._data.bucket.get(bucket)) {
// eslint-disable-next-line no-console
console.log(`Updating bucket ${bucket} with inflight ${inflight}`);
this._data.bucket.set(bucket, {
current: this._data.bucket.get(bucket).current,
nonCurrent: this._data.bucket.get(bucket).nonCurrent,
Expand Down Expand Up @@ -95,6 +97,21 @@ class Scuba {
this._server.close();
}

setInflightAsCapacity(bucketName) {
if (!this.supportsInflight) {
return;
}
this._data.bucket.forEach((value, key) => {
if (key.startsWith(`${bucketName}_`)) {
// eslint-disable-next-line no-param-reassign
value.current += value.inflight;
// eslint-disable-next-line no-param-reassign
value.inflight = 0;
this._data.bucket.set(key, value);
}
});
}

getInflightsForBucket(bucketName) {
let inflightCount = 0;
this._data.bucket.forEach((value, key) => {
Expand Down

0 comments on commit f527769

Please sign in to comment.