Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add S3ObjectLambdaEvent #536

Merged
merged 8 commits into from
Jan 6, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
83 changes: 83 additions & 0 deletions events/README_S3_Object_Lambda.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,83 @@
# Sample Function

The following is a sample class and Lambda function that receives Amazon S3 Object Lambda event record data as an input and returns an object metadata output.

```go

// main.go
package main

import (
"context"
"crypto/md5"
"encoding/hex"
"encoding/json"
"io/ioutil"
"net/http"
"github.com/aws/aws-lambda-go/lambda"
"github.com/aws/aws-lambda-go/events"
"github.com/aws/aws-sdk-go-v2/config"
"github.com/aws/aws-sdk-go-v2/service/s3"
)

func handler(ctx context.Context, event events.S3ObjectLambdaEvent) error {
url := event.GetObjectContext.InputS3Url
resp, err := http.Get(url)
if err != nil {
return err
}
defer resp.Body.Close()
bodyBytes, err := ioutil.ReadAll(resp.Body)
if err != nil {
return err
}
transformedObject := TransformedObject{
Metadata: Metadata{
Length: len(bodyBytes),
Md5: toMd5(bodyBytes),
},
}
jsonData, err := json.Marshal(transformedObject)
if err != nil {
return err
}
cfg, err := config.LoadDefaultConfig(context.TODO())
if err != nil {
return err
}
svc := s3.NewFromConfig(cfg)
input := &s3.WriteGetObjectResponseInput{
RequestRoute: &event.GetObjectContext.OutputRoute,
RequestToken: &event.GetObjectContext.OutputToken,
Body: strings.NewReader(string(jsonData)),
}
res, err := svc.WriteGetObjectResponse(ctx, input)
if err != nil {
return err
}
fmt.Printf("%v", res)
return nil
}

func toMd5(data []byte) string {
hasher := md5.New()
hasher.Write(data)
hash := hasher.Sum(nil)

return hex.EncodeToString(hash)
}

type TransformedObject struct {
Metadata Metadata `json:"metadata"`
}

type Metadata struct {
Length int `json:"length"`
Md5 string `json:"md5"`
}

func main() {
lambda.Start(handler)
}

```
64 changes: 64 additions & 0 deletions events/s3_object_lambda.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,64 @@
package events

type S3ObjectLambdaEvent struct {
XAmzRequestID string `json:"xAmzRequestId"`
GetObjectContext *S3ObjectLambdaGetObjectContext `json:"getObjectContext,omitempty"`
ListObjectsContext *S3ObjectLambdaListObjectsContext `json:"listObjectsContext,omitempty"`
ListObjectsV2Context *S3ObjectLambdaListObjectsV2Context `json:"listObjectsV2Context,omitempty"`
HeadObjectContext *S3ObjectLambdaHeadObjectContext `json:"headObjectContext,omitempty"`
Comment on lines +5 to +8
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Looking for another reference for these fields. The documentation I found so far only shows getObjectContext

https://docs.aws.amazon.com/AmazonS3/latest/userguide/olap-event-context.html

Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

PR for the rust event calavera/aws-lambda-events#127 didn't contain additional docs for List/Head object.

Guess I gotta check this by hand, or ask the service team for a pointer.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

My bad! I forgot that I referenced that exact documentation when reviewing one of your previous commits! #536 (comment)

Configuration S3ObjectLambdaConfiguration `json:"configuration"`
UserRequest S3ObjectLambdaUserRequest `json:"userRequest"`
UserIdentity S3ObjectLambdaUserIdentity `json:"userIdentity"`
ProtocolVersion string `json:"protocolVersion"`
}

type S3ObjectLambdaGetObjectContext struct {
InputS3URL string `json:"inputS3Url"`
OutputRoute string `json:"outputRoute"`
OutputToken string `json:"outputToken"`
}

type S3ObjectLambdaListObjectsContext struct {
InputS3URL string `json:"inputS3Url"`
}

type S3ObjectLambdaListObjectsV2Context struct {
InputS3URL string `json:"inputS3Url"`
}

type S3ObjectLambdaHeadObjectContext struct {
InputS3URL string `json:"inputS3Url"`
}

type S3ObjectLambdaConfiguration struct {
AccessPointARN string `json:"accessPointArn"`
SupportingAccessPointARN string `json:"supportingAccessPointArn"`
Payload string `json:"payload"`
}

type S3ObjectLambdaUserRequest struct {
URL string `json:"url"`
Headers map[string]string `json:"headers"`
}

type S3ObjectLambdaUserIdentity struct {
Type string `json:"type"`
PrincipalID string `json:"principalId"`
ARN string `json:"arn"`
AccountID string `json:"accountId"`
AccessKeyID string `json:"accessKeyId"`
SessionContext *S3ObjectLambdaSessionContext `json:"sessionContext,omitempty"`
}

type S3ObjectLambdaSessionContext struct {
Attributes map[string]string `json:"attributes"`
SessionIssuer *S3ObjectLambdaSessionIssuer `json:"sessionIssuer,omitempty"`
}

type S3ObjectLambdaSessionIssuer struct {
Type string `json:"type"`
PrincipalID string `json:"principalId"`
ARN string `json:"arn"`
AccountID string `json:"accountId"`
UserName string `json:"userName"`
}
44 changes: 44 additions & 0 deletions events/s3_object_lambda_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,44 @@
package events

import (
"encoding/json"
"testing"

"github.com/aws/aws-lambda-go/events/test"
"github.com/stretchr/testify/assert"
)

func TestS3ObjectLambdaEventMarshaling(t *testing.T) {
tests := []struct {
file string
}{
{"./testdata/s3-object-lambda-event-get-object-iam.json"},
{"./testdata/s3-object-lambda-event-get-object-assumed-role.json"},
{"./testdata/s3-object-lambda-event-head-object-iam.json"},
{"./testdata/s3-object-lambda-event-list-objects-iam.json"},
{"./testdata/s3-object-lambda-event-list-objects-v2-iam.json"},
}

for _, tc := range tests {
tc := tc
t.Run(tc.file, func(t *testing.T) {
inputJSON := test.ReadJSONFromFile(t, tc.file)

var inputEvent S3ObjectLambdaEvent
if err := json.Unmarshal(inputJSON, &inputEvent); err != nil {
t.Errorf("could not unmarshal event. details: %v", err)
}

outputJSON, err := json.Marshal(inputEvent)
if err != nil {
t.Errorf("could not marshal event. details: %v", err)
}

assert.JSONEq(t, string(inputJSON), string(outputJSON))
})
}
}

func TestS3ObjectLambdaMarshalingMalformedJson(t *testing.T) {
test.TestMalformedJson(t, S3ObjectLambdaEvent{})
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,42 @@
{
"xAmzRequestId": "requestId",
"getObjectContext": {
"inputS3Url": "https://my-s3-ap-111122223333.s3-accesspoint.us-east-1.amazonaws.com/example?X-Amz-Security-Token=<snip>",
"outputRoute": "io-use1-001",
"outputToken": "OutputToken"
},
"configuration": {
"accessPointArn": "arn:aws:s3-object-lambda:us-east-1:111122223333:accesspoint/example-object-lambda-ap",
"supportingAccessPointArn": "arn:aws:s3:us-east-1:111122223333:accesspoint/example-ap",
"payload": "{}"
},
"userRequest": {
"url": "https://object-lambda-111122223333.s3-object-lambda.us-east-1.amazonaws.com/example",
"headers": {
"Host": "object-lambda-111122223333.s3-object-lambda.us-east-1.amazonaws.com",
"Accept-Encoding": "identity",
"X-Amz-Content-SHA256": "e3b0c44298fc1example"
}
},
"userIdentity": {
"type": "AssumedRole",
"principalId": "principalId",
"arn": "arn:aws:sts::111122223333:assumed-role/Admin/example",
"accountId": "111122223333",
"accessKeyId": "accessKeyId",
"sessionContext": {
"attributes": {
"mfaAuthenticated": "false",
"creationDate": "Wed Mar 10 23:41:52 UTC 2021"
},
"sessionIssuer": {
"type": "Role",
"principalId": "principalId",
"arn": "arn:aws:iam::111122223333:role/Admin",
"accountId": "111122223333",
"userName": "Admin"
}
}
},
"protocolVersion": "1.00"
}
29 changes: 29 additions & 0 deletions events/testdata/s3-object-lambda-event-get-object-iam.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,29 @@
{
"xAmzRequestId": "requestId",
"getObjectContext": {
"inputS3Url": "https://my-s3-ap-111122223333.s3-accesspoint.us-east-1.amazonaws.com/example?X-Amz-Security-Token=<snip>",
"outputRoute": "io-use1-001",
"outputToken": "OutputToken"
},
"configuration": {
"accessPointArn": "arn:aws:s3-object-lambda:us-east-1:111122223333:accesspoint/example-object-lambda-ap",
"supportingAccessPointArn": "arn:aws:s3:us-east-1:111122223333:accesspoint/example-ap",
"payload": "{}"
},
"userRequest": {
"url": "https://object-lambda-111122223333.s3-object-lambda.us-east-1.amazonaws.com/example",
"headers": {
"Host": "object-lambda-111122223333.s3-object-lambda.us-east-1.amazonaws.com",
"Accept-Encoding": "identity",
"X-Amz-Content-SHA256": "e3b0c44298fc1example"
}
},
"userIdentity": {
"type": "IAMUser",
"principalId": "principalId",
"arn": "arn:aws:iam::111122223333:user/username",
"accountId": "111122223333",
"accessKeyId": "accessKeyId"
},
"protocolVersion": "1.00"
}
27 changes: 27 additions & 0 deletions events/testdata/s3-object-lambda-event-head-object-iam.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,27 @@
{
"xAmzRequestId": "requestId",
"headObjectContext": {
"inputS3Url": "https://my-s3-ap-111122223333.s3-accesspoint.us-east-1.amazonaws.com/example?X-Amz-Security-Token=<snip>"
},
"configuration": {
"accessPointArn": "arn:aws:s3-object-lambda:us-east-1:111122223333:accesspoint/example-object-lambda-ap",
"supportingAccessPointArn": "arn:aws:s3:us-east-1:111122223333:accesspoint/example-ap",
"payload": "{}"
},
"userRequest": {
"url": "https://object-lambda-111122223333.s3-object-lambda.us-east-1.amazonaws.com/example",
"headers": {
"Host": "object-lambda-111122223333.s3-object-lambda.us-east-1.amazonaws.com",
"Accept-Encoding": "identity",
"X-Amz-Content-SHA256": "e3b0c44298fc1example"
}
},
"userIdentity": {
"type": "IAMUser",
"principalId": "principalId",
"arn": "arn:aws:iam::111122223333:user/username",
"accountId": "111122223333",
"accessKeyId": "accessKeyId"
},
"protocolVersion": "1.01"
}
27 changes: 27 additions & 0 deletions events/testdata/s3-object-lambda-event-list-objects-iam.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,27 @@
{
"xAmzRequestId": "requestId",
"listObjectsContext": {
"inputS3Url": "https://my-s3-ap-111122223333.s3-accesspoint.us-east-1.amazonaws.com/example?X-Amz-Security-Token=<snip>"
},
"configuration": {
"accessPointArn": "arn:aws:s3-object-lambda:us-east-1:111122223333:accesspoint/example-object-lambda-ap",
"supportingAccessPointArn": "arn:aws:s3:us-east-1:111122223333:accesspoint/example-ap",
"payload": "{}"
},
"userRequest": {
"url": "https://object-lambda-111122223333.s3-object-lambda.us-east-1.amazonaws.com/example",
"headers": {
"Host": "object-lambda-111122223333.s3-object-lambda.us-east-1.amazonaws.com",
"Accept-Encoding": "identity",
"X-Amz-Content-SHA256": "e3b0c44298fc1example"
}
},
"userIdentity": {
"type": "IAMUser",
"principalId": "principalId",
"arn": "arn:aws:iam::111122223333:user/username",
"accountId": "111122223333",
"accessKeyId": "accessKeyId"
},
"protocolVersion": "1.01"
}
27 changes: 27 additions & 0 deletions events/testdata/s3-object-lambda-event-list-objects-v2-iam.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,27 @@
{
"xAmzRequestId": "requestId",
"listObjectsV2Context": {
"inputS3Url": "https://my-s3-ap-111122223333.s3-accesspoint.us-east-1.amazonaws.com/example?X-Amz-Security-Token=<snip>"
},
"configuration": {
"accessPointArn": "arn:aws:s3-object-lambda:us-east-1:111122223333:accesspoint/example-object-lambda-ap",
"supportingAccessPointArn": "arn:aws:s3:us-east-1:111122223333:accesspoint/example-ap",
"payload": "{}"
},
"userRequest": {
"url": "https://object-lambda-111122223333.s3-object-lambda.us-east-1.amazonaws.com/example",
"headers": {
"Host": "object-lambda-111122223333.s3-object-lambda.us-east-1.amazonaws.com",
"Accept-Encoding": "identity",
"X-Amz-Content-SHA256": "e3b0c44298fc1example"
}
},
"userIdentity": {
"type": "IAMUser",
"principalId": "principalId",
"arn": "arn:aws:iam::111122223333:user/username",
"accountId": "111122223333",
"accessKeyId": "accessKeyId"
},
"protocolVersion": "1.01"
}