source-aws-sns

connectors/source-aws-sns/Dockerfile

@@ -0,0 +1,5 @@
+FROM openjdk:8-jre-alpine
+WORKDIR /vance
+COPY target/aws-sns-source-1.0-SNAPSHOT-jar-with-dependencies.jar /vance
+CMD ["java", "-jar", "./aws-sns-source-1.0-SNAPSHOT-jar-with-dependencies.jar"]
+EXPOSE 8082

connectors/source-aws-sns/README.md

@@ -0,0 +1,184 @@
+# AWS SNS Source
+
+## Overview
+AWS SNS(Simple Notification Service) Source is a Vance Source Connector which subscribe to the SNS topic and receive messages published to the topic, and then transform them into CloudEvents and deliver them to the target URL. 
+Push is adopted by AWS SNS to deliver messages from SNS topics to the endpoints. Therefore, AWS SNS Source Connector should subscribe the SNS topics and start an endpoint to receive messages from the SNS topics. Application-to-application subscribers supported by SNS include http/https endpoints, Amazon Kinesis Data Firehose, Amazon SQS and AWS Lambda. AWS SNS Source Connector of Vance support http/https endpoints to receive messages. 
+
+## User Guidelines
+
+## Adapter
+Adapter convert events into CloudEvents.
+AWS SNS may send a subscription confirmation, notification, or unsubscribe confirmation message to your HTTP/HTTPS endpoints.
+Each section below describes how to transform SNS messages events into CloudEvents.
+Subscription confirmation
+### Subscription Confirmation
+
+| CloudEvents Attribute | Value                                           |
+| :-------------------- | :---------------------------------------------- |
+| `id`                  | "x-amz-sns-message-id" value |
+| `source`              | "x-amz-sns-topic-arn" value |
+| `specversion`         | `1.0`                                           |
+| `type`                | `com.amazonaws.sns.` + "x-amz-sns-message-type" value    |
+| `datacontenttype`     | `application/json`         |
+| `dataschema`          | Omit                                            |
+| `subject`             | Omit                        |
+| `time`                | "Timestamp" value                               |
+| `data`                | HTTP payload                                       |
+
+### Notification
+
+| CloudEvents Attribute | Value                                           |
+| :-------------------- | :---------------------------------------------- |
+| `id`                  | "x-amz-sns-message-id" value |
+| `source`              | "x-amz-sns-subscription-arn" value |
+| `specversion`         | `1.0`                                           |
+| `type`                | `com.amazonaws.sns.` + "x-amz-sns-message-type" value    |
+| `datacontenttype`     | `application/json`         |
+| `dataschema`          | Omit                                            |
+| `subject`             | "Subject" value (if present)                    |
+| `time`                | "Timestamp" value                               |
+| `data`                | HTTP payload                                       |
+
+### Unsubscribe Confirmation
+
+| CloudEvents Attribute | Value                                           |
+| :-------------------- | :---------------------------------------------- |
+| `id`                  | "x-amz-sns-message-id" value |
+| `source`              | "x-amz-sns-subscription-arn" value |
+| `specversion`         | `1.0`                                           |
+| `type`                | `com.amazonaws.sns.` + "x-amz-sns-message-type" value    |
+| `datacontenttype`     | `application/json`         |
+| `dataschema`          | Omit                                            |
+| `subject`             | Omit                    |
+| `time`                | "Timestamp" value                               |
+| `data`                | HTTP payload                                       |
+
+Original SNS message pushed to http/https endpoints looks like:
+```HTTP
+ POST / HTTP/1.1
+x-amz-sns-message-type: Notification
+x-amz-sns-message-id: 22b80b92-fdea-4c2c-8f9d-bdfb0c7bf324
+x-amz-sns-topic-arn: arn:aws:sns:us-west-2:123456789012:MyTopic
+x-amz-sns-subscription-arn: arn:aws:sns:us-west-2:123456789012:MyTopic:c9135db0-26c4-47ec-8998-413945fb5a96
+Content-Length: 773
+Content-Type: text/plain; charset=UTF-8
+Host: myhost.example.com
+Connection: Keep-Alive
+User-Agent: Amazon Simple Notification Service Agent
+
+{
+  "Type" : "Notification",
+  "MessageId" : "22b80b92-fdea-4c2c-8f9d-bdfb0c7bf324",
+  "TopicArn" : "arn:aws:sns:us-west-2:123456789012:MyTopic",
+  "Subject" : "My First Message",
+  "Message" : "Hello world!",
+  "Timestamp" : "2012-05-02T00:54:06.655Z",
+  "SignatureVersion" : "1",
+  "Signature" : "EXAMPLEw6JRN...",
+  "SigningCertURL" : "https://sns.us-west-2.amazonaws.com/SimpleNotificationService-f3ecfb7224c7233fe7bb5f59f96de52f.pem",
+  "UnsubscribeURL" : "https://sns.us-west-2.amazonaws.com/?Action=Unsubscribe&SubscriptionArn=arn:aws:sns:us-west-2:123456789012:MyTopic:c9135db0-26c4-47ec-8998-413945fb5a96"
+}
+```
+A transformed CloudEvent looks like:
+```JSON
+CloudEvent:{
+  "id" : "22b80b92-fdea-4c2c-8f9d-bdfb0c7bf324",
+  "source" : "arn:aws:sns:us-west-2:123456789012:MyTopic:c9135db0-26c4-47ec-8998-413945fb5a96",
+  "specversion" : "V1",
+  "type" : "com.amazonaws.sns.Notification",
+  "datacontenttype" : "application/json",
+  "subject" : "My First Message",
+  "time" : "2022-08-18T06:00:04.638Z",
+  "data" : {
+    "Type" : "Notification",
+    "MessageId" : "22b80b92-fdea-4c2c-8f9d-bdfb0c7bf324",
+    "TopicArn" : "arn:aws:sns:us-west-2:123456789012:MyTopic",
+    "Subject" : "My First Message",
+    "Message" : "Hello world!",
+    "Timestamp" : "2012-05-02T00:54:06.655Z",
+    "SignatureVersion" : "1",
+    "Signature" : "EXAMPLEw6JRN...",
+    "SigningCertURL" : "https://sns.us-west-2.amazonaws.com/SimpleNotificationService-f3ecfb7224c7233fe7bb5f59f96de52f.pem",
+	"UnsubscribeURL" : "https://sns.us-west-2.amazonaws.com/?Action=Unsubscribe&SubscriptionArn=arn:aws:sns:us-west-2:123456789012:MyTopic:c9135db0-26c4-47ec-8998-413945fb5a96"
+  }
+}
+```
+## AWS SNS Source Connector Configs
+Users can specify their configs by either setting environments variables or mount a config.json to /vance/config/config.json when they run the connector. Find examples of setting configs [here](https://github.com/linkall-labs/vance-docs/blob/main/docs/connector.md). 
+AWS SNS Source Connector requires users to provide SNS TopicArn, target url, host url, port listened of endpoints and the protocol(chosen in http and https).
+### Config Fields of the GitHub Source
+|  Configs    |  Description    																  |  Example    			  |  Required    |
+|  :----:     |  :----:         																  |  :----:     			  |  :----:      |
+|  v_target   |  v_target is used to specify the target URL HTTP Source will send CloudEvents to  |  "http://localhost:8081"  |  YES  		 |
+|  v_port     |  the port of http/http endpoints to receive SNS messages					  |  "8080"	                  |  YES         |
+|  v_host     |  the url of http/https endpoints				  |  "http://xxx.xxx.xxx:8082"	                  |  YES         |
+|  TopicArn     |  the arn of the SNS topic					  |  "arn:aws:sns:us-west-2:843378899134:Testxxxx"	                  |  YES         |
+|  protocol     |  the protocol used to subscribe SNS topic					  |  "http"	                  |  YES         |
+The config.json looks like:
+```JSON
+{
+  "TopicArn": "arn:aws:sns:us-west-2:843378899134:Testxxx",
+  "v_target": "http://8.142.xxx.xx:8080",
+  "v_port": "8082",
+  "v_host": "http://8.142.xxx.xx:8082",
+  "protocol": "http"
+}
+```
+## AWS SNS Source Connector Secrets
+Users should set their sensitive data Base64 encoded in a secret file. And mount your local secret file to /vance/secret/secret.json when you run the connector.
+### Encode your sensitive data
+```Bash
+$ echo -n ABCDEFG | base64
+QUJDREVGRw==
+```
+Replace 'ABCDEFG' with your sensitive data.
+### Set your local secret file
+```Bash
+$ cat secret.json
+{
+   "awsAccessKeyID": "${awsAccessKeyID}",
+   "awsSecretAccessKey": "${awsSecretAccessKey}"
+}
+```
+| Secrets   | Description                                                                     | Example                 | Required                 |
+|:----------|:--------------------------------------------------------------------------------|:------------------------|:------------------------|
+| awsAccessKeyID  | `awsAccessKeyID` is the Access key ID of your aws credential. | "BASE64VALUEOFYOURACCESSKEY=" |**YES** |
+| awsSecretAccessKey    | `awsSecretAccessKey` is the Secret access key of your aws credential. | "BASE64VALUEOFYOURSECRETKEY="                  |**YES** |
+
+## Amazon SNS message delivery retries
+If the target endpoint of your SNS Source Connector does not give the successful respond back, SNS Source Connector
+won't give respond to AWS SNS either. Then by default, AWS SNS will retry sending this message three times. 
+You can also configure the retry policy of the subscription corresponding to the SNS source connector yourself. Further, you can configure the SQS dead letter queue for the subscription corresponding to the SNS source connector, and then retry sending the dead letter back to SNS topic.
+
+## AWS SNS Source Connector Image
+>    
+### Run the SNS-source image in a container
+Touch the config.json and secret.json in a directory, and mount your local config file and secret file to specific positions with -v flags.
+```Bash
+docker run -v $(pwd)/secret.json:/vance/secret/secret.json -v $(pwd)/config.json:/vance/config/config.json -p 8082:8082 aws-sns-source
+```
+You can start a CloudEvents Display Sink Connector of Vance, pull the image from:
+```Bash
+docker.io/vancehub/display
+```
+Run the image with -p flags to configure the explosure port, and you can see CloudEvents delivered from AWS SNS Source Connector.
+
+## Local Development
+
+You can run the source codes of the AWS-SQS Source locally as well.
+
+### Building via Maven
+
+```shell
+$ cd connectors/source-aws-sqs
+$ mvn clean package
+```
+
+### Running via Maven
+
+```shell
+$ mvn exec:java -Dexec.mainClass="com.linkall.source.aws.sqs.Entrance"
+```
+
+⚠️ NOTE: For better local development and test, the connector can also read configs from `main/resources/config.json`. So, you don't need to 
+declare any environment variables or mount a config file to `/vance/config/config.json`. Same logic applies to `main/resources/secret.json` as well.

connectors/source-aws-sns/pom.xml

@@ -0,0 +1,91 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <modelVersion>4.0.0</modelVersion>
+
+    <groupId>org.example</groupId>
+    <artifactId>aws-sns-source</artifactId>
+    <version>1.0-SNAPSHOT</version>
+
+    <build>
+        <plugins>
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-compiler-plugin</artifactId>
+                <version>3.8.1</version>
+                <configuration>
+                    <source>1.8</source>
+                    <target>1.8</target>
+                </configuration>
+            </plugin>
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-assembly-plugin</artifactId>
+                <executions>
+                    <execution>
+                        <phase>package</phase>
+                        <goals>
+                            <goal>single</goal>
+                        </goals>
+                        <configuration>
+                            <archive>
+                                <manifest>
+                                    <mainClass>
+                                        com.linkall.source.aws.sns.Entrance
+                                    </mainClass>
+                                </manifest>
+                            </archive>
+                            <descriptorRefs>
+                                <descriptorRef>jar-with-dependencies</descriptorRef>
+                            </descriptorRefs>
+                        </configuration>
+                    </execution>
+                </executions>
+            </plugin>
+        </plugins>
+    </build>
+
+    <properties>
+        <maven.compiler.source>8</maven.compiler.source>
+        <maven.compiler.target>8</maven.compiler.target>
+    </properties>
+    <dependencies>
+
+        <dependency>
+            <groupId>com.linkall</groupId>
+            <artifactId>cdk-java</artifactId>
+            <version>0.2.1</version>
+        </dependency>
+        <dependency>
+            <groupId>io.netty</groupId>
+            <artifactId>netty-codec-http2</artifactId>
+            <version>4.1.59.Final</version>
+        </dependency>
+
+        <dependency>
+            <groupId>com.amazonaws</groupId>
+            <artifactId>aws-java-sdk-sns</artifactId>
+            <version>1.12.277</version>
+        </dependency>
+
+        <dependency>
+            <groupId>software.amazon.awssdk</groupId>
+            <artifactId>sns</artifactId>
+            <version>2.17.46</version>
+            <exclusions>
+                <exclusion>
+                    <groupId>io.netty</groupId>
+                    <artifactId>netty-codec-http2</artifactId>
+                </exclusion>
+            </exclusions>
+        </dependency>
+
+        <dependency>
+            <groupId>commons-lang</groupId>
+            <artifactId>commons-lang</artifactId>
+            <version>2.6</version>
+        </dependency>
+    </dependencies>
+
+</project>

connectors/source-aws-sns/src/main/java/com/linkall/source/aws/sns/Entrance.java

@@ -0,0 +1,9 @@
+package com.linkall.source.aws.sns;
+
+import com.linkall.vance.core.VanceApplication;
+
+public class Entrance {
+    public static void main(String[] args) {
+        VanceApplication.run(SnsSource.class);
+    }
+}

connectors/source-aws-sns/src/main/java/com/linkall/source/aws/sns/SnsAdapter.java

@@ -0,0 +1,43 @@
+package com.linkall.source.aws.sns;
+
+import com.linkall.vance.core.Adapter2;
+import io.cloudevents.CloudEvent;
+import io.cloudevents.core.builder.CloudEventBuilder;
+import io.vertx.core.buffer.Buffer;
+import io.vertx.core.http.HttpServerRequest;
+import io.vertx.core.json.JsonObject;
+import org.apache.commons.lang.StringUtils;
+
+import java.net.URI;
+import java.time.OffsetDateTime;
+
+public class SnsAdapter implements Adapter2<HttpServerRequest, Buffer> {
+
+    public static final CloudEventBuilder template = CloudEventBuilder.v1();
+
+    @Override
+    public CloudEvent adapt(HttpServerRequest httpServerRequest, Buffer buffer) {
+        JsonObject jsonObject = buffer.toJsonObject();
+        template.withId(httpServerRequest.getHeader("X-Amz-Sns-Message-Id"));
+        String subscriptionArn = httpServerRequest.getHeader("X-Amz-Sns-Subscription-Arn");
+        URI uri = null;
+        if(StringUtils.isBlank(subscriptionArn)){
+            uri = URI.create(httpServerRequest.getHeader("X-Amz-Sns-Topic-Arn"));
+        }else{
+            uri = URI.create(subscriptionArn);
+        }
+        template.withSource(uri);
+        String type = "com.amazonaws.sns."+jsonObject.getString("Type");
+        template.withType(type)
+                .withDataContentType("application/json");
+        String subject = jsonObject.getString("Subject");
+        if(!StringUtils.isBlank(subject)){
+            template.withSubject(subject);
+        }
+        String timeStamp = jsonObject.getString("Timestamp");
+        OffsetDateTime time = OffsetDateTime.parse(timeStamp);
+        template.withTime(time);
+        template.withData(buffer.getBytes());
+        return template.build();
+    }
+}