Skip to content

Commit 6d6d17b

Browse files
naveenpaul1naveenpaul1
authored
Merge pull request #9213 from naveenpaul1/replication_sdk_fix
SDK | Fix log replication sdk issue
2 parents 7521471 + e8e5927 commit 6d6d17b

File tree

4 files changed

+36
-35
lines changed

4 files changed

+36
-35
lines changed

‎src/server/bg_services/replication_log_parser.js‎

Lines changed: 23 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -1,17 +1,16 @@
11
/* Copyright (C) 2016 NooBaa */
22
'use strict';
33

4-
const AWS = require('aws-sdk');
54
const system_store = require('../system_services/system_store').get_instance();
65
const replication_store = require('../system_services/replication_store').instance();
76
const cloud_utils = require('../../util/cloud_utils');
8-
const http_utils = require('../../util/http_utils');
97
const pool_server = require('../system_services/pool_server');
108
const dbg = require('../../util/debug_module')(__filename);
119
const config = require('../../../config');
1210
const moment = require('moment');
1311
const { LogsQueryClient, LogsQueryResultStatus } = require("@azure/monitor-query");
1412
const { ClientSecretCredential } = require("@azure/identity");
13+
const noobaa_s3_client = require('../../sdk/noobaa_s3_client/noobaa_s3_client');
1514

1615
/**
1716
* get_log_candidates will return an object which contains the log candidates
@@ -61,7 +60,8 @@ async function get_aws_log_candidates(source_bucket_id, rule_id, replication_con
6160
}
6261

6362
const next_log_data = await _aws_get_next_log(s3, logs_bucket, next_log_entry.Contents[0].Key);
64-
aws_parse_log_object(logs, next_log_data, sync_deletions, obj_prefix_filter);
63+
const log_string = await next_log_data.Body.transformToString();
64+
await aws_parse_log_object(logs, log_string, sync_deletions, obj_prefix_filter);
6565

6666
dbg.log1("get_aws_log_candidates: parsed logs ", logs);
6767

@@ -227,6 +227,13 @@ function create_candidates(logs) {
227227
return candidates;
228228
}
229229

230+
/**
231+
* _aws_get_next_log will get the log object
232+
* @param {S3} s3
233+
* @param {string} logs_bucket
234+
* @param {string} logs_prefix
235+
* @param {*} continuation_token
236+
*/
230237
async function aws_get_next_log_entry(s3, logs_bucket, logs_prefix, continuation_token) {
231238
let start_after = logs_prefix;
232239
if (start_after && !start_after.endsWith('/')) {
@@ -243,7 +250,7 @@ async function aws_get_next_log_entry(s3, logs_bucket, logs_prefix, continuation
243250

244251
try {
245252
dbg.log2('log_parser aws_get_next_log_entry: params:', params);
246-
const res = await s3.listObjectsV2(params).promise();
253+
const res = await s3.listObjectsV2(params);
247254
dbg.log1('log_parser aws_get_next_log_entry: finished successfully ', res);
248255
return res;
249256

@@ -255,7 +262,7 @@ async function aws_get_next_log_entry(s3, logs_bucket, logs_prefix, continuation
255262

256263
/**
257264
* _aws_get_next_log will get the log object
258-
* @param {AWS.S3} s3
265+
* @param {S3} s3
259266
* @param {string} bucket
260267
* @param {string} key
261268
*/
@@ -266,7 +273,7 @@ async function _aws_get_next_log(s3, bucket, key) {
266273
Bucket: bucket,
267274
Key: key,
268275
ResponseContentType: 'json'
269-
}).promise();
276+
});
270277

271278
dbg.log1('log_parser _aws_get_next_log: finished successfully ', res);
272279
return res;
@@ -280,12 +287,11 @@ async function _aws_get_next_log(s3, bucket, key) {
280287
/**
281288
* aws_parse_log_object will parse the log object and will return an array of candidates
282289
* @param {nb.ReplicationLogs} logs - Log array
283-
* @param {*} log_object - AWS log object
290+
* @param {String} log_string - AWS log object
284291
* @param {boolean} sync_deletions - Whether deletions should be synced or not
285292
* @param {string} obj_prefix_filter - Object prefix filter
286293
*/
287-
function aws_parse_log_object(logs, log_object, sync_deletions, obj_prefix_filter) {
288-
const log_string = log_object.Body.toString();
294+
async function aws_parse_log_object(logs, log_string, sync_deletions, obj_prefix_filter) {
289295
const log_array = log_string.split("\n");
290296

291297
for (const line of log_array) {
@@ -362,25 +368,20 @@ function _get_source_bucket_azure_connection(source_bucket_id) {
362368

363369
function _get_source_bucket_aws_connection(source_bucket_id, aws_log_replication_info) {
364370
const source_bucket = system_store.data.get_by_id(source_bucket_id);
365-
const logs_location = aws_log_replication_info.logs_location;
366-
const { logs_bucket } = logs_location;
367371

368372
const s3_resource_connection_info =
369373
pool_server.get_namespace_resource_extended_info(source_bucket.namespace.write_resource.resource);
370-
371-
const agent = s3_resource_connection_info.endpoint_type === 'AWS' ?
372-
http_utils.get_default_agent(s3_resource_connection_info.endpoint) :
373-
http_utils.get_unsecured_agent(s3_resource_connection_info.endpoint);
374-
375-
const s3 = new AWS.S3({
376-
params: { Bucket: logs_bucket },
374+
const s3 = noobaa_s3_client.get_s3_client_v3_params({
377375
endpoint: s3_resource_connection_info.endpoint,
378-
accessKeyId: s3_resource_connection_info.access_key.unwrap(),
379-
secretAccessKey: s3_resource_connection_info.secret_key.unwrap(),
376+
credentials: {
377+
accessKeyId: s3_resource_connection_info.access_key.unwrap(),
378+
secretAccessKey: s3_resource_connection_info.secret_key.unwrap(),
379+
},
380+
forcePathStyle: true,
381+
region: config.DEFAULT_REGION,
380382
signatureVersion: cloud_utils.get_s3_endpoint_signature_ver(s3_resource_connection_info.endpoint,
381383
s3_resource_connection_info.auth_method),
382-
s3ForcePathStyle: true,
383-
httpOptions: { agent }
384+
requestHandler: noobaa_s3_client.get_requestHandler_with_suitable_agent(s3_resource_connection_info.endpoint),
384385
});
385386

386387
return s3;

‎src/server/utils/replication_utils.js‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -108,8 +108,8 @@ async function get_object_md(bucket_name, key, s3, version_id) {
108108
dbg.log1('get_object_md: finished successfully', head);
109109
return head;
110110
} catch (err) {
111-
dbg.error('get_object_md: error:', err);
112-
if (err.code === 'NotFound') return;
111+
dbg.error('get_object_md: error.name: ', err?.name, ' error: ', err);
112+
if (err?.name === 'NotFound') return;
113113
throw err;
114114
}
115115
}

‎src/test/unit_tests/internal/test_bucket_log_based_replication.test.js‎

Lines changed: 10 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -19,17 +19,17 @@ describe('AWS S3 server log parsing tests', () => {
1919
// Pagination test
2020
it('Test AWS S3 server log parsing for BATCH.DELETE and REST.PUT actions', async () => {
2121
const logs = [];
22-
const example_log = { Body: `
22+
const example_log = `
2323
aaa test.bucket [13/Feb/2023:15:08:28 +0000] 1.1.1.1 arn:aws:iam::111:user/user AAA BATCH.DELETE.OBJECT test - 204 - - 1 - - - - - AAA SigV4 ECDHE-RSA-AES128-GCM-SHA256 AuthHeader s3.us-east-2.amazonaws.com TLSv1.2 - -
2424
aaa test.bucket [13/Feb/2023:15:08:28 +0000] 1.1.1.1 arn:aws:iam::111:user/user AAA BATCH.DELETE.OBJECT test.js - 204 - - 1 - - - - - AAA SigV4 ECDHE-RSA-AES128-GCM-SHA256 AuthHeader s3.us-east-2.amazonaws.com TLSv1.2 - -
2525
aaa test.bucket [13/Feb/2023:16:08:56 +0000] 0.0.0.0 arn:aws:iam::111:user/user AAA REST.PUT.OBJECT code2 "PUT /test.bucket/code2?X-Amz-Security-Token=AAAAAAAAAAAAAAA=20230213T160856Z&X-Amz-AAAAAA HTTP/1.1" 200 - - 1 1 1 "https://s3.console.aws.amazon.com/s3/upload/test.bucket?region=us-east-2" "AAA/5.0 (AAA 1.1; AAA; AAA) AAA/1.1 (KHTML, like Gecko) AAA/1.1 AAA/1.1" - AAAA SigV4 ECDHE-RSA-AES128-GCM-SHA256 QueryString s3.us-east-2.amazonaws.com TLSv1.2 - -
2626
aaa test.bucket [13/Feb/2023:15:08:28 +0000] 1.1.1.1 arn:aws:iam::111:user/user AAA BATCH.DELETE.OBJECT test2 - 204 - - 1 - - - - - AAA SigV4 ECDHE-RSA-AES128-GCM-SHA256 AuthHeader s3.us-east-2.amazonaws.com TLSv1.2 - -
2727
aaa test.bucket [13/Feb/2023:15:08:28 +0000] 1.1.1.1 arn:aws:iam::111:user/user AAA BATCH.DELETE.OBJECT testfile.js - 204 - - 1 - - - - - AAA SigV4 ECDHE-RSA-AES128-GCM-SHA256 AuthHeader s3.us-east-2.amazonaws.com TLSv1.2 - -
2828
aaa test.bucket [13/Feb/2023:15:25:00 +0000] 0.0.0.0 arn:aws:iam::111:user/user AAA REST.PUT.OBJECT empty "PUT /test.bucket/empty?X-Amz-Security-Token=AAAAAAAAAAAAAAA=20230213T152500Z&X-Amz-AAAAAA HTTP/1.1" 200 - - 1 1 1 "https://s3.console.aws.amazon.com/s3/upload/test.bucket?region=us-east-2" "AAA/5.0 (AAA 1.1; AAA; AAA) AAA/1.1 (KHTML, like Gecko) AAA/1.1 AAA/1.1" - AAAA SigV4 ECDHE-RSA-AES128-GCM-SHA256 QueryString s3.us-east-2.amazonaws.com TLSv1.2 - -
2929
aaa test.bucket [13/Feb/2023:15:08:28 +0000] 1.1.1.1 arn:aws:iam::111:user/user AAA BATCH.DELETE.OBJECT text.txt - 204 - - 1 - - - - - AAA SigV4 ECDHE-RSA-AES128-GCM-SHA256 AuthHeader s3.us-east-2.amazonaws.com TLSv1.2 - -
30-
` };
30+
`;
3131
const action_dictionary = { 'test': 'delete', 'test.js': 'delete', 'code2': 'copy', 'test2': 'delete', 'testfile.js': 'delete', 'empty': 'copy', 'text.txt': 'delete' };
32-
log_parser.aws_parse_log_object(logs, example_log, true, '');
32+
await log_parser.aws_parse_log_object(logs, example_log, true, '');
3333
// Make sure the test doesn't pass in case the parsing fails
3434
expect(logs.length).toEqual(Object.keys(action_dictionary).length);
3535
// Make sure all expected actions are mapped to the appropriate keys
@@ -38,7 +38,7 @@ describe('AWS S3 server log parsing tests', () => {
3838
});
3939
// Test with sync_deletions set to false
4040
logs.length = 0;
41-
log_parser.aws_parse_log_object(logs, example_log, false, '');
41+
await log_parser.aws_parse_log_object(logs, example_log, false, '');
4242
// Delete all action_dictionary keys whose value is delete
4343
Object.keys(action_dictionary).forEach(key => {
4444
if (action_dictionary[key] === 'delete') {
@@ -50,12 +50,12 @@ describe('AWS S3 server log parsing tests', () => {
5050

5151
it('Test AWS S3 server log parsing when a DELETE is logged before a PUT, but occurs after it', async () => {
5252
const logs = [];
53-
const example_log = { Body: `
53+
const example_log = `
5454
aaa test.bucket [13/Feb/2023:19:08:28 +0000] 1.1.1.1 arn:aws:iam::111:user/user AAA BATCH.DELETE.OBJECT test - 204 - - 1 - - - - - AAA SigV4 ECDHE-RSA-AES128-GCM-SHA256 AuthHeader s3.us-east-2.amazonaws.com TLSv1.2 - -
5555
aaa test.bucket [13/Feb/2023:09:08:28 +0000] 1.1.1.1 arn:aws:iam::111:user/user AAA BATCH.DELETE.OBJECT other_obj - 204 - - 1 - - - - - AAA SigV4 ECDHE-RSA-AES128-GCM-SHA256 AuthHeader s3.us-east-2.amazonaws.com TLSv1.2 - -
5656
aaa test.bucket [13/Feb/2023:09:08:56 +0000] 0.0.0.0 arn:aws:iam::111:user/user AAA REST.PUT.OBJECT test "PUT /test.bucket/test?X-Amz-Security-Token=AAAAAAAAAAAAAAA=20230213T160856Z&X-Amz-AAAAAA HTTP/1.1" 200 - - 1 1 1 "https://s3.console.aws.amazon.com/s3/upload/test.bucket?region=us-east-2" "AAA/5.0 (AAA 1.1; AAA; AAA) AAA/1.1 (KHTML, like Gecko) AAA/1.1 AAA/1.1" - AAAA SigV4 ECDHE-RSA-AES128-GCM-SHA256 QueryString s3.us-east-2.amazonaws.com TLSv1.2 - -
57-
` };
58-
log_parser.aws_parse_log_object(logs, example_log, true, '');
57+
`;
58+
await log_parser.aws_parse_log_object(logs, example_log, true, '');
5959
const candidates = log_parser.create_candidates(logs);
6060
// DELETE log should be the latest log present inside the candidate, as candidate storing only latest log per key
6161
expect(candidates.test.action).toEqual('delete');
@@ -246,7 +246,7 @@ describe('AWS S3 server logs parsing/processing tests', () => {
246246
const logs = [];
247247
const src_bucket = "src-bucket";
248248
const dst_bucket = "dst-bucket";
249-
const example_log = {Body: `
249+
const example_log = `
250250
aaa test.bucket [13/Feb/2023:19:08:28 +0000] 1.1.1.1 arn:aws:iam::111:user/user AAA BATCH.DELETE.OBJECT test - 204 - - 1 - - - - - AAA SigV4 ECDHE-RSA-AES128-GCM-SHA256 AuthHeader s3.us-east-2.amazonaws.com TLSv1.2 - -
251251
aaa test.bucket [13/Feb/2023:09:08:28 +0000] 1.1.1.1 arn:aws:iam::111:user/user AAA BATCH.DELETE.OBJECT other_obj - 204 - - 1 - - - - - AAA SigV4 ECDHE-RSA-AES128-GCM-SHA256 AuthHeader s3.us-east-2.amazonaws.com TLSv1.2 - -
252252
aaa test.bucket [13/Feb/2023:19:08:28 +0000] 0.0.0.0 arn:aws:iam::111:user/user AAA REST.PUT.OBJECT test "PUT /test.bucket/code2?X-Amz-Security-Token=AAAAAAAAAAAAAAA=20230213T160856Z&X-Amz-AAAAAA HTTP/1.1" 200 - - 1 1 1 "https://s3.console.aws.amazon.com/s3/upload/test.bucket?region=us-east-2" "AAA/5.0 (AAA 1.1; AAA; AAA) AAA/1.1 (KHTML, like Gecko) AAA/1.1 AAA/1.1" - AAAA SigV4 ECDHE-RSA-AES128-GCM-SHA256 QueryString s3.us-east-2.amazonaws.com TLSv1.2 - -
@@ -261,10 +261,10 @@ describe('AWS S3 server logs parsing/processing tests', () => {
261261
aaa test.bucket [12/Feb/2023:15:08:28 +0000] 1.1.1.1 arn:aws:iam::111:user/user AAA BATCH.DELETE.OBJECT test - 204 - - 1 - - - - - AAA SigV4 ECDHE-RSA-AES128-GCM-SHA256 AuthHeader s3.us-east-2.amazonaws.com TLSv1.2 - -
262262
aaa test.bucket [13/Feb/2023:15:08:28 +0000] 1.1.1.1 arn:aws:iam::111:user/user AAA BATCH.DELETE.OBJECT test.js - 204 - - 1 - - - - - AAA SigV4 ECDHE-RSA-AES128-GCM-SHA256 AuthHeader s3.us-east-2.amazonaws.com TLSv1.2 - -
263263
aaa test.bucket [13/Feb/2023:16:08:56 +0000] 0.0.0.0 arn:aws:iam::111:user/user AAA REST.PUT.OBJECT code2 "PUT /test.bucket/code2?X-Amz-Security-Token=AAAAAAAAAAAAAAA=20230213T160856Z&X-Amz-AAAAAA HTTP/1.1" 200 - - 1 1 1 "https://s3.console.aws.amazon.com/s3/upload/test.bucket?region=us-east-2" "AAA/5.0 (AAA 1.1; AAA; AAA) AAA/1.1 (KHTML, like Gecko) AAA/1.1 AAA/1.1" - AAAA SigV4 ECDHE-RSA-AES128-GCM-SHA256 QueryString s3.us-east-2.amazonaws.com TLSv1.2 - - aaa test.bucket [13/Feb/2023:15:08:28 +0000] 1.1.1.1 arn:aws:iam::111:user/user AAA BATCH.DELETE.OBJECT testfile.js - 204 - - 1 - - - - - AAA SigV4 ECDHE-RSA-AES128-GCM-SHA256 AuthHeader s3.us-east-2.amazonaws.com TLSv1.2 - - aaa test.bucket [13/Feb/2023:15:08:28 +0000] 1.1.1.1 arn:aws:iam::111:user/user AAA BATCH.DELETE.OBJECT text.txt - 204 - - 1 - - - - - AAA SigV4 ECDHE-RSA-AES128-GCM-SHA256 AuthHeader s3.us-east-2.amazonaws.com TLSv1.2 - -
264-
`};
264+
`;
265265

266266

267-
log_parser.aws_parse_log_object(logs, example_log, true);
267+
await log_parser.aws_parse_log_object(logs, example_log, true, '');
268268
// Make sure the test doesn't pass in case the parsing fails
269269
expect(logs.length).toEqual(14);
270270

‎src/util/cloud_utils.js‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -185,7 +185,7 @@ function set_noobaa_s3_connection(sys) {
185185
dbg.error('set_noobaa_s3_connection: temporary error: invalid noobaa s3 connection details');
186186
return;
187187
}
188-
const s3_client = new S3({
188+
const s3_client = noobaa_s3_client.get_s3_client_v3_params({
189189
endpoint: endpoint,
190190
credentials: {
191191
accessKeyId: access_key,

0 commit comments

Comments
 (0)