Skip to content

Commit

Permalink
CLDSRV-518 Duplication of version ID in metadata
Browse files Browse the repository at this point in the history
  • Loading branch information
@nicolas2bert
nicolas2bert committed Mar 12, 2024
1 parent a0e5257 commit 77d390d
Show file tree
Hide file tree
Showing 3 changed files with 173 additions and 49 deletions.
5 changes: 5 additions & 0 deletions lib/routes/routeBackbeat.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -497,6 +497,11 @@ function putMetadata(request, response, bucketInfo, objMd, log, callback) {
// If no new versioned objects are added for given object(s), they look like
// standalone master keys.
versioning = false;
// Delete the version id from the version metadata payload to prevent issues
// with creating a non-version object (versioning set to false) that includes a version id.
// For example, this version ID might come from a null version of a suspended bucket being
// replicated to this bucket.
delete omVal.versionId;
} else {
const versioningConf = bucketInfo.getVersioningConfiguration();
// The purpose of this condition is to address situations in which
Expand Down
156 changes: 107 additions & 49 deletions tests/functional/raw-node/test/routes/routeBackbeat.js
View file
Original file line number Diff line number Diff line change
@@ -1,9 +1,8 @@
const assert = require('assert');
const async = require('async');
const crypto = require('crypto');
const { models, versioning } = require('arsenal');
const { versioning } = require('arsenal');
const versionIdUtils = versioning.VersionID;
const { ObjectMD } = models;

const { makeRequest, makeBackbeatRequest } = require('../../utils/makeRequest');
const BucketUtility = require('../../../aws-node-sdk/lib/utility/bucket-util');
Expand Down Expand Up @@ -87,17 +86,15 @@ function checkVersionData(s3, bucket, objectKey, versionId, dataValue, done) {
}

function updateStorageClass(data, storageClass) {
let parsedBody;
let result;
try {
parsedBody = JSON.parse(data.body);
const parsedBody = JSON.parse(JSON.parse(data.body).Body);
parsedBody['x-amz-storage-class'] = storageClass;
result = JSON.stringify(parsedBody);
} catch (err) {
return { error: err };
}
const { result, error } = ObjectMD.createFromBlob(parsedBody.Body);
if (error) {
return { error };
}
result.setAmzStorageClass(storageClass);

return { result };
}

Expand Down Expand Up @@ -202,11 +199,11 @@ describeSkipIfAWS('backbeat routes', () => {

it('should update metadata of a current null version', done => {
let objMD;
return async.series([
next => s3.putObject({ Bucket: bucket, Key: keyName, Body: new Buffer(testData) }, next),
next => s3.putBucketVersioning({ Bucket: bucket, VersioningConfiguration: { Status: 'Enabled' } },
next),
next => makeBackbeatRequest({
return async.series({
putObject: next => s3.putObject({ Bucket: bucket, Key: keyName, Body: new Buffer(testData) }, next),
enableVersioningSource: next => s3.putBucketVersioning(
{ Bucket: bucket, VersioningConfiguration: { Status: 'Enabled' } }, next),
getMetadata: next => makeBackbeatRequest({
method: 'GET',
resourceType: 'metadata',
bucket,
Expand All @@ -226,7 +223,7 @@ describeSkipIfAWS('backbeat routes', () => {
objMD = result;
return next();
}),
next => makeBackbeatRequest({
putMetadata: next => makeBackbeatRequest({
method: 'PUT',
resourceType: 'metadata',
bucket,
Expand All @@ -235,19 +232,37 @@ describeSkipIfAWS('backbeat routes', () => {
versionId: 'null',
},
authCredentials: backbeatAuthCredentials,
requestBody: objMD.getSerialized(),
requestBody: objMD,
}, next),
next => s3.headObject({ Bucket: bucket, Key: keyName, VersionId: 'null' }, next),
next => s3.listObjectVersions({ Bucket: bucket }, next),
], (err, data) => {
headObject: next => s3.headObject(
{ Bucket: bucket, Key: keyName, VersionId: 'null' }, next),
getMetadataAfter: next => makeBackbeatRequest({
method: 'GET',
resourceType: 'metadata',
bucket,
objectKey: keyName,
queryObj: {
versionId: 'null',
},
authCredentials: backbeatAuthCredentials,
}, next),
listObjectVersions: next => s3.listObjectVersions({ Bucket: bucket }, next),
}, (err, results) => {
if (err) {
return done(err);
}
const headObjectRes = data[4];

const headObjectRes = results.headObject;
assert.strictEqual(headObjectRes.VersionId, 'null');
assert.strictEqual(headObjectRes.StorageClass, storageClass);

const listObjectVersionsRes = data[5];
const getMetadataAfterRes = results.getMetadataAfter;
const objMDAfter = JSON.parse(getMetadataAfterRes.body).Body;
const expectedMd = JSON.parse(objMD);
expectedMd.isNull = true; // TODO remove the line once CLDSRV-509 is fixed
assert.deepStrictEqual(JSON.parse(objMDAfter), expectedMd);

const listObjectVersionsRes = results.listObjectVersions;
const { Versions } = listObjectVersionsRes;

assert.strictEqual(Versions.length, 1);
Expand All @@ -261,18 +276,20 @@ describeSkipIfAWS('backbeat routes', () => {
it('should update metadata of a non-current null version', done => {
let objMD;
let expectedVersionId;
return async.series([
next => s3.putObject({ Bucket: bucket, Key: keyName, Body: new Buffer(testData) }, next),
next => s3.putBucketVersioning({ Bucket: bucket, VersioningConfiguration: { Status: 'Enabled' } },
next),
next => s3.putObject({ Bucket: bucket, Key: keyName, Body: new Buffer(testData) }, (err, data) => {
return async.series({
putObjectInitial: next => s3.putObject(
{ Bucket: bucket, Key: keyName, Body: new Buffer(testData) }, next),
enableVersioning: next => s3.putBucketVersioning(
{ Bucket: bucket, VersioningConfiguration: { Status: 'Enabled' } }, next),
putObjectAgain: next => s3.putObject(
{ Bucket: bucket, Key: keyName, Body: new Buffer(testData) }, (err, data) => {
if (err) {
return next(err);
}
expectedVersionId = data.VersionId;
return next();
}),
next => makeBackbeatRequest({
getMetadata: next => makeBackbeatRequest({
method: 'GET',
resourceType: 'metadata',
bucket,
Expand All @@ -292,7 +309,7 @@ describeSkipIfAWS('backbeat routes', () => {
objMD = result;
return next();
}),
next => makeBackbeatRequest({
putMetadata: next => makeBackbeatRequest({
method: 'PUT',
resourceType: 'metadata',
bucket,
Expand All @@ -301,23 +318,36 @@ describeSkipIfAWS('backbeat routes', () => {
versionId: 'null',
},
authCredentials: backbeatAuthCredentials,
requestBody: objMD.getSerialized(),
requestBody: objMD,
}, next),
next => s3.headObject({ Bucket: bucket, Key: keyName, VersionId: 'null' }, next),
next => s3.listObjectVersions({ Bucket: bucket }, next),
], (err, data) => {
headObject: next => s3.headObject({ Bucket: bucket, Key: keyName, VersionId: 'null' }, next),
getMetadataAfter: next => makeBackbeatRequest({
method: 'GET',
resourceType: 'metadata',
bucket,
objectKey: keyName,
queryObj: {
versionId: 'null',
},
authCredentials: backbeatAuthCredentials,
}, next),
listObjectVersions: next => s3.listObjectVersions({ Bucket: bucket }, next),
}, (err, results) => {
if (err) {
return done(err);
}
const headObjectRes = data[5];
const headObjectRes = results.headObject;
assert.strictEqual(headObjectRes.VersionId, 'null');
assert.strictEqual(headObjectRes.StorageClass, storageClass);

const listObjectVersionsRes = data[6];
const getMetadataAfterRes = results.getMetadataAfter;
const objMDAfter = JSON.parse(getMetadataAfterRes.body).Body;
assert.deepStrictEqual(JSON.parse(objMDAfter), JSON.parse(objMD));

const listObjectVersionsRes = results.listObjectVersions;
const { Versions } = listObjectVersionsRes;

assert.strictEqual(Versions.length, 2);

const currentVersion = Versions.find(v => v.IsLatest);
assertVersionHasNotBeenUpdated(currentVersion, expectedVersionId);

Expand Down Expand Up @@ -365,9 +395,19 @@ describeSkipIfAWS('backbeat routes', () => {
versionId: 'null',
},
authCredentials: backbeatAuthCredentials,
requestBody: objMD.getSerialized(),
requestBody: objMD,
}, next),
headObject: next => s3.headObject({ Bucket: bucket, Key: keyName, VersionId: 'null' }, next),
getMetadataAfter: next => makeBackbeatRequest({
method: 'GET',
resourceType: 'metadata',
bucket,
objectKey: keyName,
queryObj: {
versionId: 'null',
},
authCredentials: backbeatAuthCredentials,
}, next),
listObjectVersions: next => s3.listObjectVersions({ Bucket: bucket }, next),
}, (err, results) => {
if (err) {
Expand All @@ -377,6 +417,10 @@ describeSkipIfAWS('backbeat routes', () => {
assert.strictEqual(headObjectRes.VersionId, 'null');
assert.strictEqual(headObjectRes.StorageClass, storageClass);

const getMetadataAfterRes = results.getMetadataAfter;
const objMDAfter = JSON.parse(getMetadataAfterRes.body).Body;
assert.deepStrictEqual(JSON.parse(objMDAfter), JSON.parse(objMD));

const listObjectVersionsRes = results.listObjectVersions;
const { Versions } = listObjectVersionsRes;

Expand Down Expand Up @@ -430,9 +474,19 @@ describeSkipIfAWS('backbeat routes', () => {
versionId: 'null',
},
authCredentials: backbeatAuthCredentials,
requestBody: objMD.getSerialized(),
requestBody: objMD,
}, next),
headObject: next => s3.headObject({ Bucket: bucket, Key: keyName, VersionId: 'null' }, next),
getMetadataAfter: next => makeBackbeatRequest({
method: 'GET',
resourceType: 'metadata',
bucket,
objectKey: keyName,
queryObj: {
versionId: 'null',
},
authCredentials: backbeatAuthCredentials,
}, next),
listObjectVersions: next => s3.listObjectVersions({ Bucket: bucket }, next),
}, (err, results) => {
if (err) {
Expand All @@ -442,6 +496,10 @@ describeSkipIfAWS('backbeat routes', () => {
assert.strictEqual(headObjectRes.VersionId, 'null');
assert.strictEqual(headObjectRes.StorageClass, storageClass);

const getMetadataAfterRes = results.getMetadataAfter;
const objMDAfter = JSON.parse(getMetadataAfterRes.body).Body;
assert.deepStrictEqual(JSON.parse(objMDAfter), JSON.parse(objMD));

const listObjectVersionsRes = results.listObjectVersions;
const { Versions } = listObjectVersionsRes;

Expand Down Expand Up @@ -487,7 +545,7 @@ describeSkipIfAWS('backbeat routes', () => {
versionId: 'null',
},
authCredentials: backbeatAuthCredentials,
requestBody: objMD.getSerialized(),
requestBody: objMD,
}, next),
next => s3.headObject({ Bucket: bucket, Key: keyName }, next),
next => s3.listObjectVersions({ Bucket: bucket }, next),
Expand Down Expand Up @@ -549,7 +607,7 @@ describeSkipIfAWS('backbeat routes', () => {
versionId: 'null',
},
authCredentials: backbeatAuthCredentials,
requestBody: objMD.getSerialized(),
requestBody: objMD,
}, next),
next => s3.headObject({ Bucket: bucket, Key: keyName }, next),
next => s3.listObjectVersions({ Bucket: bucket }, next),
Expand Down Expand Up @@ -612,7 +670,7 @@ describeSkipIfAWS('backbeat routes', () => {
versionId: 'null',
},
authCredentials: backbeatAuthCredentials,
requestBody: objMD.getSerialized(),
requestBody: objMD,
}, next),
next => s3.headObject({ Bucket: bucket, Key: keyName }, next),
next => s3.listObjectVersions({ Bucket: bucket }, next),
Expand Down Expand Up @@ -683,7 +741,7 @@ describeSkipIfAWS('backbeat routes', () => {
versionId: 'null',
},
authCredentials: backbeatAuthCredentials,
requestBody: objMD.getSerialized(),
requestBody: objMD,
}, next),
next => s3.headObject({ Bucket: bucket, Key: keyName }, next),
next => s3.listObjectVersions({ Bucket: bucket }, next),
Expand Down Expand Up @@ -748,7 +806,7 @@ describeSkipIfAWS('backbeat routes', () => {
versionId: 'null',
},
authCredentials: backbeatAuthCredentials,
requestBody: objMD.getSerialized(),
requestBody: objMD,
}, next),
next => s3.headObject({ Bucket: bucket, Key: keyName }, next),
next => s3.listObjectVersions({ Bucket: bucket }, next),
Expand Down Expand Up @@ -808,7 +866,7 @@ describeSkipIfAWS('backbeat routes', () => {
versionId: 'null',
},
authCredentials: backbeatAuthCredentials,
requestBody: objMD.getSerialized(),
requestBody: objMD,
}, next),
next => s3.headObject({ Bucket: bucket, Key: keyName, VersionId: 'null' }, next),
next => s3.listObjectVersions({ Bucket: bucket }, next),
Expand Down Expand Up @@ -869,7 +927,7 @@ describeSkipIfAWS('backbeat routes', () => {
versionId: 'null',
},
authCredentials: backbeatAuthCredentials,
requestBody: objMD.getSerialized(),
requestBody: objMD,
}, next),
next => s3.putObject({ Bucket: bucket, Key: keyName, Body: new Buffer(testData) }, next),
next => s3.headObject({ Bucket: bucket, Key: keyName, VersionId: 'null' }, next),
Expand Down Expand Up @@ -932,7 +990,7 @@ describeSkipIfAWS('backbeat routes', () => {
versionId: 'null',
},
authCredentials: backbeatAuthCredentials,
requestBody: objMD.getSerialized(),
requestBody: objMD,
}, next),
next => s3.putBucketVersioning({ Bucket: bucket, VersioningConfiguration: { Status: 'Enabled' } },
next),
Expand Down Expand Up @@ -1012,7 +1070,7 @@ describeSkipIfAWS('backbeat routes', () => {
versionId: 'null',
},
authCredentials: backbeatAuthCredentials,
requestBody: objMD.getSerialized(),
requestBody: objMD,
}, next),
next => s3.headObject({ Bucket: bucket, Key: keyName, VersionId: 'null' }, next),
next => s3.listObjectVersions({ Bucket: bucket }, next),
Expand Down Expand Up @@ -1087,7 +1145,7 @@ describeSkipIfAWS('backbeat routes', () => {
versionId: 'null',
},
authCredentials: backbeatAuthCredentials,
requestBody: objMD.getSerialized(),
requestBody: objMD,
}, next),
next => s3.headObject({ Bucket: bucket, Key: keyName, VersionId: 'null' }, next),
next => s3.listObjectVersions({ Bucket: bucket }, next),
Expand Down Expand Up @@ -1159,7 +1217,7 @@ describeSkipIfAWS('backbeat routes', () => {
versionId: 'null',
},
authCredentials: backbeatAuthCredentials,
requestBody: objMD.getSerialized(),
requestBody: objMD,
}, next),
next => s3.putObject({ Bucket: bucket, Key: keyName, Body: new Buffer(testData) }, next),
next => s3.headObject({ Bucket: bucket, Key: keyName, VersionId: 'null' }, next),
Expand Down Expand Up @@ -1233,7 +1291,7 @@ describeSkipIfAWS('backbeat routes', () => {
versionId: 'null',
},
authCredentials: backbeatAuthCredentials,
requestBody: objMD.getSerialized(),
requestBody: objMD,
}, next),
next => s3.putBucketVersioning({ Bucket: bucket, VersioningConfiguration: { Status: 'Enabled' } },
next),
Expand Down
Loading

0 comments on commit 77d390d

Please sign in to comment.