diff --git a/src/api/bucket_api.js b/src/api/bucket_api.js index 94abdef0c1..036979b67b 100644 --- a/src/api/bucket_api.js +++ b/src/api/bucket_api.js @@ -779,9 +779,9 @@ module.exports = { method: 'PUT', params: { type: 'object', - required: ['source_bucket', 'log_bucket', 'log_prefix'], + required: ['name', 'log_bucket', 'log_prefix'], properties: { - source_bucket: { + name: { $ref: 'common_api#/definitions/bucket_name' }, log_bucket: { @@ -801,9 +801,9 @@ module.exports = { method: 'DELETE', params: { type: 'object', - required: ['source_bucket'], + required: ['name'], properties: { - source_bucket: { + name: { $ref: 'common_api#/definitions/bucket_name' }, }, @@ -817,9 +817,9 @@ module.exports = { method: 'GET', params: { type: 'object', - required: ['source_bucket'], + required: ['name'], properties: { - source_bucket: { + name: { $ref: 'common_api#/definitions/bucket_name' }, }, @@ -827,7 +827,7 @@ module.exports = { reply: { type: 'object', properties: { - source_bucket: { + name: { $ref: 'common_api#/definitions/bucket_name' }, log_bucket: { @@ -1068,9 +1068,9 @@ module.exports = { logging: { type: 'object', - required: ['source_bucket', 'log_bucket', 'log_prefix'], + required: ['name', 'log_bucket', 'log_prefix'], properties: { - source_bucket: { + name: { $ref: 'common_api#/definitions/bucket_name', }, log_bucket: { diff --git a/src/deploy/NVA_build/NooBaa.Dockerfile b/src/deploy/NVA_build/NooBaa.Dockerfile index 5825aa5ece..602cb5b79e 100644 --- a/src/deploy/NVA_build/NooBaa.Dockerfile +++ b/src/deploy/NVA_build/NooBaa.Dockerfile @@ -120,6 +120,7 @@ RUN chmod 775 /noobaa_init_files && \ chgrp -R 0 /noobaa_init_files/ && \ chmod -R g=u /noobaa_init_files/ + COPY --from=server_builder /kubectl /usr/local/bin/kubectl COPY --from=server_builder ./noobaa_init_files/kube_pv_chown /noobaa_init_files RUN mkdir -m 777 /root/node_modules && \ diff --git a/src/endpoint/s3/s3_bucket_logging.js b/src/endpoint/s3/s3_bucket_logging.js new file mode 100644 index 0000000000..7545803e1f --- /dev/null +++ b/src/endpoint/s3/s3_bucket_logging.js @@ -0,0 +1,66 @@ +/* Copyright (C) 2016 NooBaa */ +'use strict'; + +const dbg = require('../../util/debug_module')(__filename); +const http_utils = require('../../util/http_utils'); + + +async function send_bucket_op_logs(req, res) { + if (req.params && req.params.bucket) { + const bucket_info = await req.object_sdk.read_bucket_sdk_config_info(req.params.bucket); + dbg.log2("read_bucket_sdk_config_info = ", bucket_info); + + if (is_bucket_logging_enabled(bucket_info)) { + dbg.log2("Bucket logging is enabled for Bucket : ", req.params.bucket); + endpoint_bucket_op_logs(req.op_name, req, res, bucket_info); + } + } +} + + +function is_bucket_logging_enabled(source_bucket) { + + if (!source_bucket || !source_bucket.bucket_info.logging) { + return false; + } + return true; +} + +function endpoint_bucket_op_logs(op_name, req, res, source_bucket) { + + dbg.log2("Sending op logs for op name = ", op_name); + // 1 - Get all the information to be logged in a log message. + // 2 - Format it and send it to log bucket/syslog. + const s3_log = get_bucket_log_record(op_name, source_bucket, req, res); + dbg.log1("Bucket operation logs = ", s3_log); + +} + +function get_bucket_log_record(op_name, source_bucket, req, res) { + + const client_ip = http_utils.parse_client_ip(req); + let status_code; + if (res && res.statusCode) { + status_code = res.statusCode; + } + const log = { + op: req.method, + bucket_owner: source_bucket.bucket_owner, + source_bucket: req.params.bucket, + object_key: req.originalUrl, + log_bucket: source_bucket.bucket_info.logging.log_bucket, + remote_ip: client_ip, + request_uri: req.originalUrl, + http_status: status_code, + request_id: req.request_id + }; + + return log; +} + + +exports.is_bucket_logging_enabled = is_bucket_logging_enabled; +exports.endpoint_bucket_op_logs = endpoint_bucket_op_logs; +exports.get_bucket_log_record = get_bucket_log_record; +exports.send_bucket_op_logs = send_bucket_op_logs; + diff --git a/src/endpoint/s3/s3_rest.js b/src/endpoint/s3/s3_rest.js index 675c1438e4..6d9c560051 100755 --- a/src/endpoint/s3/s3_rest.js +++ b/src/endpoint/s3/s3_rest.js @@ -8,6 +8,7 @@ const dbg = require('../../util/debug_module')(__filename); const s3_ops = require('./ops'); const S3Error = require('./s3_errors').S3Error; const s3_bucket_policy_utils = require('./s3_bucket_policy_utils'); +const s3_logging = require('./s3_bucket_logging'); const time_utils = require('../../util/time_utils'); const http_utils = require('../../util/http_utils'); const signature_utils = require('../../util/signature_utils'); @@ -116,8 +117,6 @@ async function handle_request(req, res) { usage_report.s3_usage_info.total_calls += 1; usage_report.s3_usage_info[op_name] = (usage_report.s3_usage_info[op_name] || 0) + 1; - - if (req.query && req.query.versionId) { const caching = await req.object_sdk.read_bucket_sdk_caching_info(req.params.bucket); if (caching) { @@ -148,6 +147,8 @@ async function handle_request(req, res) { const reply = await op.handler(req, res); http_utils.send_reply(req, res, reply, options); collect_bucket_usage(op, req, res); + await s3_logging.send_bucket_op_logs(req); + } async function populate_request_additional_info_or_redirect(req) { diff --git a/src/sdk/bucketspace_nb.js b/src/sdk/bucketspace_nb.js index a00f553d35..f1235ac6ab 100644 --- a/src/sdk/bucketspace_nb.js +++ b/src/sdk/bucketspace_nb.js @@ -137,6 +137,28 @@ class BucketSpaceNB { name: params.name }); } + //////////////////// + // BUCKET LOGGING // + //////////////////// + + async put_bucket_logging(params) { + return this.rpc_client.bucket.put_bucket_logging({ + name: params.name, + logging: params.logging + }); + } + + async delete_bucket_logging(params) { + return this.rpc_client.bucket.delete_bucket_logging({ + name: params.name + }); + } + + async get_bucket_logging(req) { + return this.rpc_client.bucket.get_bucket_logging({ + name: req.params.bucket + }); + } /////////////////////// // BUCKET ENCRYPTION // diff --git a/src/sdk/nb.d.ts b/src/sdk/nb.d.ts index 76c74bda4c..e4186029f7 100644 --- a/src/sdk/nb.d.ts +++ b/src/sdk/nb.d.ts @@ -25,7 +25,7 @@ type NodeType = 'BLOCK_STORE_GOOGLE' | 'BLOCK_STORE_FS' | 'ENDPOINT_S3'; - + type S3Response = ServerResponse; type S3Request = IncomingMessage & { object_sdk: ObjectSDK; @@ -830,6 +830,10 @@ interface BucketSpace { delete_bucket_tagging(params: object): Promise; get_bucket_tagging(params: object): Promise; + put_bucket_logging(params: object): Promise; + delete_bucket_logging(params: object): Promise; + get_bucket_logging(params: object): Promise; + put_bucket_encryption(params: object): Promise; get_bucket_encryption(params: object): Promise; delete_bucket_encryption(params: object): Promise; diff --git a/src/sdk/object_sdk.js b/src/sdk/object_sdk.js index b4c10f55cd..6ec1a40656 100644 --- a/src/sdk/object_sdk.js +++ b/src/sdk/object_sdk.js @@ -160,6 +160,11 @@ class ObjectSDK { return bucket.namespace; } + async read_bucket_sdk_config_info(name) { + const { bucket } = await bucket_namespace_cache.get_with_cache({ sdk: this, name }); + return bucket; + } + async read_bucket_sdk_caching_info(name) { try { const { bucket } = await bucket_namespace_cache.get_with_cache({ sdk: this, name }); @@ -926,6 +931,25 @@ class ObjectSDK { return bs.get_bucket_tagging(params); } + //////////////////// + // BUCKET LOGGING // + //////////////////// + + async put_bucket_logging(params) { + const bs = this._get_bucketspace(); + return bs.put_bucket_logging(params); + } + + async delete_bucket_logging(params) { + const bs = this._get_bucketspace(); + return bs.delete_bucket_logging(params); + } + + async get_bucket_logging(req) { + const bs = this._get_bucketspace(); + return bs.get_bucket_logging(req); + } + /////////////////////// // BUCKET ENCRYPTION // /////////////////////// diff --git a/src/server/system_services/bucket_server.js b/src/server/system_services/bucket_server.js index 3b9c6749d3..41d2f65f28 100644 --- a/src/server/system_services/bucket_server.js +++ b/src/server/system_services/bucket_server.js @@ -391,27 +391,40 @@ async function delete_bucket_tagging(req) { */ async function put_bucket_logging(req) { + dbg.log0('put_bucket_logging:', req.rpc_params); const bucket = find_bucket(req); + const logging = { + "name": bucket.name, + "log_bucket": req.rpc_params.log_bucket, + "log_prefix": req.rpc_params.log_prefix + }; + await system_store.make_changes({ update: { buckets: [{ _id: bucket._id, - logging: req.rpc_params.logging + logging }] } }); } async function get_bucket_logging(req) { + dbg.log0('get_bucket_logging:', req.rpc_params); const bucket = find_bucket(req); - return { - logging: bucket.logging, + + const logging = { + "name": bucket.name, + "log_bucket": bucket.logging.log_bucket, + "log_prefix": bucket.logging.log_prefix }; + return logging; } async function delete_bucket_logging(req) { + dbg.log0('delete_bucket_logging:', req.rpc_params); const bucket = find_bucket(req); await system_store.make_changes({