Skip to content

Commit

Permalink
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
feat(logs): add distribution property to the Subscription class (aws#…
Browse files Browse the repository at this point in the history
…30423)

### Issue # (if applicable)

Closes aws#30422

### Reason for this change
Missing Property in the Subscription class.



### Description of changes
Add destination property to the Subscription class.



### Description of how you validated changes
Add unit tests and integ tests.



### Checklist
- [x] My code adheres to the [CONTRIBUTING GUIDE](https://github.com/aws/aws-cdk/blob/main/CONTRIBUTING.md) and [DESIGN GUIDELINES](https://github.com/aws/aws-cdk/blob/main/docs/DESIGN_GUIDELINES.md)

----

*By submitting this pull request, I confirm that my contribution is made under the terms of the Apache-2.0 license*
mazyu36 authored and Leonardo Gama committed Jun 11, 2024
1 parent 325f528 commit 995d239
Showing 13 changed files with 838 additions and 4 deletions.

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

Original file line number Diff line number Diff line change
@@ -0,0 +1,172 @@
{
"Resources": {
"LogGroupF5B46931": {
"Type": "AWS::Logs::LogGroup",
"Properties": {
"RetentionInDays": 731
},
"UpdateReplacePolicy": "Retain",
"DeletionPolicy": "Retain"
},
"LogGroupSubscriptionCloudWatchLogsCanPutRecords29011851": {
"Type": "AWS::IAM::Role",
"Properties": {
"AssumeRolePolicyDocument": {
"Statement": [
{
"Action": "sts:AssumeRole",
"Effect": "Allow",
"Principal": {
"Service": "logs.amazonaws.com"
}
}
],
"Version": "2012-10-17"
}
}
},
"LogGroupSubscriptionCloudWatchLogsCanPutRecordsDefaultPolicyB7125314": {
"Type": "AWS::IAM::Policy",
"Properties": {
"PolicyDocument": {
"Statement": [
{
"Action": [
"kinesis:ListShards",
"kinesis:PutRecord",
"kinesis:PutRecords"
],
"Effect": "Allow",
"Resource": {
"Fn::GetAtt": [
"Stream790BDEE4",
"Arn"
]
}
},
{
"Action": "iam:PassRole",
"Effect": "Allow",
"Resource": {
"Fn::GetAtt": [
"LogGroupSubscriptionCloudWatchLogsCanPutRecords29011851",
"Arn"
]
}
}
],
"Version": "2012-10-17"
},
"PolicyName": "LogGroupSubscriptionCloudWatchLogsCanPutRecordsDefaultPolicyB7125314",
"Roles": [
{
"Ref": "LogGroupSubscriptionCloudWatchLogsCanPutRecords29011851"
}
]
}
},
"LogGroupSubscriptionE3573E29": {
"Type": "AWS::Logs::SubscriptionFilter",
"Properties": {
"DestinationArn": {
"Fn::GetAtt": [
"Stream790BDEE4",
"Arn"
]
},
"Distribution": "Random",
"FilterName": "CustomSubscriptionFilterName",
"FilterPattern": "\"ERROR\" \"MainThread\"",
"LogGroupName": {
"Ref": "LogGroupF5B46931"
},
"RoleArn": {
"Fn::GetAtt": [
"LogGroupSubscriptionCloudWatchLogsCanPutRecords29011851",
"Arn"
]
}
},
"DependsOn": [
"LogGroupSubscriptionCloudWatchLogsCanPutRecordsDefaultPolicyB7125314"
]
},
"Stream790BDEE4": {
"Type": "AWS::Kinesis::Stream",
"Properties": {
"RetentionPeriodHours": 24,
"ShardCount": 1,
"StreamEncryption": {
"Fn::If": [
"AwsCdkKinesisEncryptedStreamsUnsupportedRegions",
{
"Ref": "AWS::NoValue"
},
{
"EncryptionType": "KMS",
"KeyId": "alias/aws/kinesis"
}
]
}
},
"UpdateReplacePolicy": "Retain",
"DeletionPolicy": "Retain"
}
},
"Conditions": {
"AwsCdkKinesisEncryptedStreamsUnsupportedRegions": {
"Fn::Or": [
{
"Fn::Equals": [
{
"Ref": "AWS::Region"
},
"cn-north-1"
]
},
{
"Fn::Equals": [
{
"Ref": "AWS::Region"
},
"cn-northwest-1"
]
}
]
}
},
"Parameters": {
"BootstrapVersion": {
"Type": "AWS::SSM::Parameter::Value<String>",
"Default": "/cdk-bootstrap/hnb659fds/version",
"Description": "Version of the CDK Bootstrap resources in this environment, automatically retrieved from SSM Parameter Store. [cdk:skip]"
}
},
"Rules": {
"CheckBootstrapVersion": {
"Assertions": [
{
"Assert": {
"Fn::Not": [
{
"Fn::Contains": [
[
"1",
"2",
"3",
"4",
"5"
],
{
"Ref": "BootstrapVersion"
}
]
}
]
},
"AssertDescription": "CDK bootstrap stack version 6 required. Please run 'cdk bootstrap' with a recent version of the CDK CLI."
}
]
}
}
}

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

Original file line number Diff line number Diff line change
@@ -0,0 +1,28 @@
import { App, Stack, StackProps } from 'aws-cdk-lib';
import { IntegTest } from '@aws-cdk/integ-tests-alpha';
import { LogGroup, FilterPattern, Distribution } from 'aws-cdk-lib/aws-logs';
import { KinesisDestination } from 'aws-cdk-lib/aws-logs-destinations';
import { Stream } from 'aws-cdk-lib/aws-kinesis';
class SubscriptionFilterDistributionIntegStack extends Stack {
constructor(scope: App, id: string, props?: StackProps) {
super(scope, id, props);

const logGroup = new LogGroup(this, 'LogGroup');

const stream = new Stream(this, 'Stream');

logGroup.addSubscriptionFilter('Subscription', {
destination: new KinesisDestination(stream),
filterPattern: FilterPattern.allTerms('ERROR', 'MainThread'),
filterName: 'CustomSubscriptionFilterName',
distribution: Distribution.RANDOM,
});
}
}

const app = new App();
const testCase = new SubscriptionFilterDistributionIntegStack(app, 'aws-cdk-subscriptionfilter-distribution-integ');

new IntegTest(app, 'integ-test', {
testCases: [testCase],
});
23 changes: 21 additions & 2 deletions packages/aws-cdk-lib/aws-logs/README.md
Original file line number Diff line number Diff line change
@@ -120,6 +120,25 @@ new logs.SubscriptionFilter(this, 'Subscription', {
});
```

When you use `KinesisDestination`, you can choose the method used to
distribute log data to the destination by setting the `distribution` property.

```ts
import * as destinations from 'aws-cdk-lib/aws-logs-destinations';
import * as kinesis from 'aws-cdk-lib/aws-kinesis';

declare const stream: kinesis.Stream;
declare const logGroup: logs.LogGroup;

new logs.SubscriptionFilter(this, 'Subscription', {
logGroup,
destination: new destinations.KinesisDestination(stream),
filterPattern: logs.FilterPattern.allTerms("ERROR", "MainThread"),
filterName: 'ErrorInMainThread',
distribution: logs.Distribution.RANDOM,
});
```

## Metric Filters

CloudWatch Logs can extract and emit metrics based on a textual log stream.
@@ -344,12 +363,12 @@ For more information, see [Protect sensitive log data with masking](https://docs

For a list of types of managed identifiers that can be audited and masked, see [Types of data that you can protect](https://docs.aws.amazon.com/AmazonCloudWatch/latest/logs/protect-sensitive-log-data-types.html).

If a new identifier is supported but not yet in the `DataIdentifiers` enum, the name of the identifier can be supplied as `name` in the constructor instead.
If a new identifier is supported but not yet in the `DataIdentifiers` enum, the name of the identifier can be supplied as `name` in the constructor instead.

To add a custom data identifier, supply a custom `name` and `regex` to the `CustomDataIdentifiers` constructor.
For more information on custom data identifiers, see [Custom data identifiers](https://docs.aws.amazon.com/AmazonCloudWatch/latest/logs/CWL-custom-data-identifiers.html).

Each policy may consist of a log group, S3 bucket, and/or Firehose delivery stream audit destination.
Each policy may consist of a log group, S3 bucket, and/or Firehose delivery stream audit destination.

Example:

23 changes: 23 additions & 0 deletions packages/aws-cdk-lib/aws-logs/lib/log-group.ts
Original file line number Diff line number Diff line change
@@ -440,6 +440,21 @@ export interface LogGroupProps {
readonly removalPolicy?: RemovalPolicy;
}

/**
* The method used to distribute log data to the destination.
*/
export enum Distribution {
/**
* Log events from the same log stream are kept together and sent to the same destination.
*/
BY_LOG_STREAM = 'ByLogStream',

/**
* Log events are distributed across the log destinations randomly.
*/
RANDOM = 'Random',
}

/**
* Define a CloudWatch Log Group
*/
@@ -573,6 +588,14 @@ export interface SubscriptionFilterOptions {
* @default Automatically generated
*/
readonly filterName?: string;

/**
* The method used to distribute log data to the destination.
* This property can only be used with KinesisDestination.
*
* @default Distribution.BY_LOG_STREAM
*/
readonly distribution?: Distribution;
}

/**
13 changes: 12 additions & 1 deletion packages/aws-cdk-lib/aws-logs/lib/subscription-filter.ts
Original file line number Diff line number Diff line change
@@ -2,7 +2,8 @@ import { Construct } from 'constructs';
import { ILogGroup, SubscriptionFilterOptions } from './log-group';
import { CfnSubscriptionFilter } from './logs.generated';
import * as iam from '../../aws-iam';
import { Resource } from '../../core';
import { KinesisDestination } from '../../aws-logs-destinations';
import { Resource, Token } from '../../core';

/**
* Interface for classes that can be the destination of a log Subscription
@@ -57,6 +58,15 @@ export class SubscriptionFilter extends Resource {
physicalName: props.filterName,
});

if (
props.distribution &&
!Token.isUnresolved(props.distribution) &&
!Token.isUnresolved(props.destination) &&
!(props.destination instanceof KinesisDestination)
) {
throw new Error('distribution property can only be used with KinesisDestination.');
}

const destProps = props.destination.bind(this, props.logGroup);

new CfnSubscriptionFilter(this, 'Resource', {
@@ -65,6 +75,7 @@ export class SubscriptionFilter extends Resource {
roleArn: destProps.role && destProps.role.roleArn,
filterPattern: props.filterPattern.logPatternString,
filterName: this.physicalName,
distribution: props.distribution,
});
}
}
41 changes: 40 additions & 1 deletion packages/aws-cdk-lib/aws-logs/test/subscriptionfilter.test.ts
Original file line number Diff line number Diff line change
@@ -1,7 +1,9 @@
import { Construct } from 'constructs';
import { Template } from '../../assertions';
import { Stream } from '../../aws-kinesis';
import { KinesisDestination } from '../../aws-logs-destinations';
import { Stack } from '../../core';
import { FilterPattern, ILogGroup, ILogSubscriptionDestination, LogGroup, SubscriptionFilter } from '../lib';
import { Distribution, FilterPattern, ILogGroup, ILogSubscriptionDestination, LogGroup, SubscriptionFilter } from '../lib';

describe('subscription filter', () => {
test('trivial instantiation', () => {
@@ -45,6 +47,43 @@ describe('subscription filter', () => {
FilterName: 'CustomSubscriptionFilterName',
});
});

test('subscription filter with KinesisDestination can have distribution set.', () => {
// GIVEN
const stack = new Stack();
const logGroup = new LogGroup(stack, 'LogGroup');

const stream = new Stream(stack, 'Stream');

// WHEN
new SubscriptionFilter(stack, 'Subscription', {
logGroup,
destination: new KinesisDestination(stream),
filterPattern: FilterPattern.literal('some pattern'),
filterName: 'CustomSubscriptionFilterName',
distribution: Distribution.RANDOM,
});

// THEN
Template.fromStack(stack).hasResourceProperties('AWS::Logs::SubscriptionFilter', {
Distribution: 'Random',
});
});

test('subscription filter with non-KinesisDestination can not have distribution set.', () => {
const stack = new Stack();
const logGroup = new LogGroup(stack, 'LogGroup');

expect(() => {
new SubscriptionFilter(stack, 'Subscription', {
logGroup,
destination: new FakeDestination(),
filterPattern: FilterPattern.literal('some pattern'),
filterName: 'CustomSubscriptionFilterName',
distribution: Distribution.RANDOM,
});
}).toThrow('distribution property can only be used with KinesisDestination.');
});
});

class FakeDestination implements ILogSubscriptionDestination {

0 comments on commit 995d239

Please sign in to comment.