From f7277839e6848933e2da7ef36bd46876e218e908 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=AF=95=E5=8D=9A?= Date: Thu, 27 Oct 2022 06:59:52 +0800 Subject: [PATCH 01/15] add rabbitmq connector --- .../connector-rabbitmq/pom.xml | 47 +++++ .../rabbitmq/client/QueueingConsumer.java | 96 +++++++++ .../rabbitmq/source/DeliveryWithSplitId.java | 14 ++ .../rabbitmq/source/RabbitMQSource.java | 86 ++++++++ .../rabbitmq/source/RabbitMQSourceReader.java | 185 ++++++++++++++++++ .../source/RabbitMQSplitReaderThread.java | 35 ++++ .../rabbitmq/split/RabbitMQSplit.java | 46 +++++ .../rabbitmq/split/RabbitMQState.java | 6 + seatunnel-connectors-v2/pom.xml | 1 + 9 files changed, 516 insertions(+) create mode 100644 seatunnel-connectors-v2/connector-rabbitmq/pom.xml create mode 100644 seatunnel-connectors-v2/connector-rabbitmq/src/main/java/org/apache/seatunnel/connectors/seatunnel/rabbitmq/client/QueueingConsumer.java create mode 100644 seatunnel-connectors-v2/connector-rabbitmq/src/main/java/org/apache/seatunnel/connectors/seatunnel/rabbitmq/source/DeliveryWithSplitId.java create mode 100644 seatunnel-connectors-v2/connector-rabbitmq/src/main/java/org/apache/seatunnel/connectors/seatunnel/rabbitmq/source/RabbitMQSource.java create mode 100644 seatunnel-connectors-v2/connector-rabbitmq/src/main/java/org/apache/seatunnel/connectors/seatunnel/rabbitmq/source/RabbitMQSourceReader.java create mode 100644 seatunnel-connectors-v2/connector-rabbitmq/src/main/java/org/apache/seatunnel/connectors/seatunnel/rabbitmq/source/RabbitMQSplitReaderThread.java create mode 100644 seatunnel-connectors-v2/connector-rabbitmq/src/main/java/org/apache/seatunnel/connectors/seatunnel/rabbitmq/split/RabbitMQSplit.java create mode 100644 seatunnel-connectors-v2/connector-rabbitmq/src/main/java/org/apache/seatunnel/connectors/seatunnel/rabbitmq/split/RabbitMQState.java diff --git a/seatunnel-connectors-v2/connector-rabbitmq/pom.xml b/seatunnel-connectors-v2/connector-rabbitmq/pom.xml new file mode 100644 index 00000000000..43ff333f20e --- /dev/null +++ b/seatunnel-connectors-v2/connector-rabbitmq/pom.xml @@ -0,0 +1,47 @@ + + + + + seatunnel-connectors-v2 + org.apache.seatunnel + ${revision} + + 4.0.0 + + connector-rabbitmq + + + 5.9.0 + + + + org.apache.seatunnel + connector-common + ${project.version} + + + com.rabbitmq + amqp-client + ${rabbitmq.version} + + + diff --git a/seatunnel-connectors-v2/connector-rabbitmq/src/main/java/org/apache/seatunnel/connectors/seatunnel/rabbitmq/client/QueueingConsumer.java b/seatunnel-connectors-v2/connector-rabbitmq/src/main/java/org/apache/seatunnel/connectors/seatunnel/rabbitmq/client/QueueingConsumer.java new file mode 100644 index 00000000000..b9bf30bf95f --- /dev/null +++ b/seatunnel-connectors-v2/connector-rabbitmq/src/main/java/org/apache/seatunnel/connectors/seatunnel/rabbitmq/client/QueueingConsumer.java @@ -0,0 +1,96 @@ +package org.apache.seatunnel.connectors.seatunnel.rabbitmq.client; + +import com.rabbitmq.client.AMQP; +import com.rabbitmq.client.Channel; +import com.rabbitmq.client.ConsumerCancelledException; +import com.rabbitmq.client.DefaultConsumer; +import com.rabbitmq.client.Delivery; +import com.rabbitmq.client.Envelope; +import com.rabbitmq.client.ShutdownSignalException; +import com.rabbitmq.utility.Utility; + +import java.io.IOException; +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.TimeUnit; + +public class QueueingConsumer extends DefaultConsumer { + private final BlockingQueue queue; + + // When this is non-null the queue is in shutdown mode and nextDelivery should + // throw a shutdown signal exception. + private volatile ShutdownSignalException shutdown; + private volatile ConsumerCancelledException cancelled; + + private static final Delivery POISON = new Delivery(null, null, null); + + public QueueingConsumer(Channel channel) { + this(channel, Integer.MAX_VALUE); + } + + public QueueingConsumer(Channel channel, int capacity) { + super(channel); + this.queue = new LinkedBlockingQueue<>(capacity); + } + + private void checkShutdown() { + if (shutdown != null) { + throw Utility.fixStackTrace(shutdown); + } + } + + private Delivery handle(Delivery delivery) { + if (delivery == POISON || delivery == null && (shutdown != null || cancelled != null)) { + if (delivery == POISON) { + queue.add(POISON); + if (shutdown == null && cancelled == null) { + throw new IllegalStateException( + "POISON in queue, but null shutdown and null cancelled. " + + "This should never happen, please report as a BUG"); + } + } + if (null != shutdown) { + throw Utility.fixStackTrace(shutdown); + } + if (null != cancelled) { + throw Utility.fixStackTrace(cancelled); + } + } + return delivery; + } + + public Delivery nextDelivery() + throws InterruptedException, ShutdownSignalException, ConsumerCancelledException { + return handle(queue.take()); + } + + public Delivery nextDelivery(long timeout) + throws InterruptedException, ShutdownSignalException, ConsumerCancelledException { + return nextDelivery(timeout, TimeUnit.MILLISECONDS); + } + + public Delivery nextDelivery(long timeout, TimeUnit unit) + throws InterruptedException, ShutdownSignalException, ConsumerCancelledException { + return handle(queue.poll(timeout, unit)); + } + + @Override + public void handleShutdownSignal(String consumerTag, ShutdownSignalException sig) { + shutdown = sig; + queue.add(POISON); + } + + @Override + public void handleCancel(String consumerTag) throws IOException { + cancelled = new ConsumerCancelledException(); + queue.add(POISON); + } + + @Override + public void handleDelivery( + String consumerTag, Envelope envelope, AMQP.BasicProperties properties, byte[] body) + throws IOException { + checkShutdown(); + this.queue.add(new Delivery(envelope, properties, body)); + } +} diff --git a/seatunnel-connectors-v2/connector-rabbitmq/src/main/java/org/apache/seatunnel/connectors/seatunnel/rabbitmq/source/DeliveryWithSplitId.java b/seatunnel-connectors-v2/connector-rabbitmq/src/main/java/org/apache/seatunnel/connectors/seatunnel/rabbitmq/source/DeliveryWithSplitId.java new file mode 100644 index 00000000000..0e7dc4ade22 --- /dev/null +++ b/seatunnel-connectors-v2/connector-rabbitmq/src/main/java/org/apache/seatunnel/connectors/seatunnel/rabbitmq/source/DeliveryWithSplitId.java @@ -0,0 +1,14 @@ +package org.apache.seatunnel.connectors.seatunnel.rabbitmq.source; + +import com.rabbitmq.client.Delivery; +import lombok.AllArgsConstructor; +import lombok.Getter; +import lombok.Setter; + +@AllArgsConstructor +@Setter +@Getter +public final class DeliveryWithSplitId { + private final Delivery delivery; + private final String splitId; +} diff --git a/seatunnel-connectors-v2/connector-rabbitmq/src/main/java/org/apache/seatunnel/connectors/seatunnel/rabbitmq/source/RabbitMQSource.java b/seatunnel-connectors-v2/connector-rabbitmq/src/main/java/org/apache/seatunnel/connectors/seatunnel/rabbitmq/source/RabbitMQSource.java new file mode 100644 index 00000000000..efbcd7c39f7 --- /dev/null +++ b/seatunnel-connectors-v2/connector-rabbitmq/src/main/java/org/apache/seatunnel/connectors/seatunnel/rabbitmq/source/RabbitMQSource.java @@ -0,0 +1,86 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.connectors.seatunnel.rabbitmq.source; + +import com.google.auto.service.AutoService; +import org.apache.seatunnel.api.common.JobContext; +import org.apache.seatunnel.api.common.PrepareFailException; +import org.apache.seatunnel.api.serialization.DeserializationSchema; +import org.apache.seatunnel.api.source.Boundedness; +import org.apache.seatunnel.api.source.SeaTunnelSource; +import org.apache.seatunnel.api.source.SourceReader; +import org.apache.seatunnel.api.source.SourceSplitEnumerator; +import org.apache.seatunnel.api.table.type.SeaTunnelRow; +import org.apache.seatunnel.api.table.type.SeaTunnelRowType; +import org.apache.seatunnel.common.constants.JobMode; +import org.apache.seatunnel.connectors.seatunnel.rabbitmq.split.RabbitMQSplit; +import org.apache.seatunnel.connectors.seatunnel.rabbitmq.split.RabbitMQState; +import org.apache.seatunnel.shade.com.typesafe.config.Config; + +@AutoService(SeaTunnelSource.class) +public class RabbitMQSource implements SeaTunnelSource { + + private DeserializationSchema deserializationSchema; + private SeaTunnelRowType typeInfo; + private JobContext jobContext; + + @Override + public Boundedness getBoundedness() { + return JobMode.BATCH.equals(jobContext.getJobMode()) ? Boundedness.BOUNDED : Boundedness.UNBOUNDED; + } + + @Override + public String getPluginName() { + return "RabbitMQ"; + } + + @Override + public void prepare(Config config) throws PrepareFailException { + + } + + @Override + public SeaTunnelRowType getProducedType() { + return this.typeInfo; + } + + @Override + public SourceReader createReader(SourceReader.Context readerContext) throws Exception { + return null; + } + + @Override + public SourceSplitEnumerator createEnumerator(SourceSplitEnumerator.Context enumeratorContext) throws Exception { + return null; + } + + @Override + public SourceSplitEnumerator restoreEnumerator(SourceSplitEnumerator.Context enumeratorContext, RabbitMQState checkpointState) throws Exception { + return null; + } + + + @Override + public void setJobContext(JobContext jobContext) { + this.jobContext = jobContext; + } + + private void setDeserialization(Config config) { + + } +} diff --git a/seatunnel-connectors-v2/connector-rabbitmq/src/main/java/org/apache/seatunnel/connectors/seatunnel/rabbitmq/source/RabbitMQSourceReader.java b/seatunnel-connectors-v2/connector-rabbitmq/src/main/java/org/apache/seatunnel/connectors/seatunnel/rabbitmq/source/RabbitMQSourceReader.java new file mode 100644 index 00000000000..a570f03409b --- /dev/null +++ b/seatunnel-connectors-v2/connector-rabbitmq/src/main/java/org/apache/seatunnel/connectors/seatunnel/rabbitmq/source/RabbitMQSourceReader.java @@ -0,0 +1,185 @@ +package org.apache.seatunnel.connectors.seatunnel.rabbitmq.source; + +import com.rabbitmq.client.*; +import lombok.extern.slf4j.Slf4j; +import org.apache.seatunnel.api.serialization.DeserializationSchema; +import org.apache.seatunnel.api.source.Collector; +import org.apache.seatunnel.api.source.SourceReader; +import org.apache.seatunnel.common.Handover; +import org.apache.seatunnel.connectors.seatunnel.pulsar.source.split.PulsarPartitionSplit; +import org.apache.seatunnel.connectors.seatunnel.rabbitmq.split.RabbitMQSplit; + +import java.io.IOException; +import java.util.*; +import java.util.stream.Collectors; + +@Slf4j +public class RabbitMQSourceReader implements SourceReader { + protected final Handover handover; + protected final DeserializationSchema deserialization; + + private boolean noMoreSplitsAssignment = false; + + protected final SourceReader.Context context; + protected final int batchSize; + + protected transient Connection connection; + protected transient Channel channel; + private final boolean usesCorrelationId; + protected transient boolean autoAck; + + private transient volatile boolean running; + + protected transient List sessionIds; + protected final Map splitStates; + protected final Set finishedSplits; + protected final SortedMap>> pendingCursorsToCommit; + protected final Map> pendingCursorsToFinish; + + public RabbitMQSourceReader(Handover handover, + DeserializationSchema deserialization, + SourceReader.Context context, + int batchSize, + boolean usesCorrelationId) { + this.handover = new Handover<>(); + this.deserialization = deserialization; + this.pendingCursorsToCommit = Collections.synchronizedSortedMap(new TreeMap<>()); + this.pendingCursorsToFinish = Collections.synchronizedSortedMap(new TreeMap<>()); + this.finishedSplits = new TreeSet<>(); + this.splitStates = new HashMap<>(); + this.context = context; + this.batchSize = batchSize; + this.usesCorrelationId = usesCorrelationId; + } + + @Override + public void open() throws Exception { + sessionIds = new ArrayList<>(64); + } + + @Override + public void close() throws IOException { + + } + + @Override + public void pollNext(Collector output) throws Exception { + for (int i = 0; i < batchSize; i++) { + Optional deliveryOptional = handover.pollNext(); + if (deliveryOptional.isPresent()) { + Delivery delivery = deliveryOptional.get().getDelivery(); + AMQP.BasicProperties properties = delivery.getProperties(); + byte[] body = delivery.getBody(); + Envelope envelope = delivery.getEnvelope(); + synchronized (output.getCheckpointLock()) { + splitStates.get(deliveryOptional.get().getSplitId()).getDeliveryTags().add(envelope.getDeliveryTag()); + deserialization.deserialize(body, output); + } + } + if (noMoreSplitsAssignment && finishedSplits.size() == splitStates.size()) { + context.signalNoMoreElement(); + break; + } + } + } + + @Override + public List snapshotState(long checkpointId) throws Exception { + List pendingSplit = splitStates.values().stream() + .map(RabbitMQSplit::copy) + .collect(Collectors.toList()); + // Perform a snapshot for these splits. + int size = pendingSplit.size(); + Map> cursors = + pendingCursorsToCommit.computeIfAbsent(checkpointId, id -> new HashMap<>(size)); + // Put the cursors of the active splits. + for (RabbitMQSplit split : pendingSplit) { + List latestConsumedId = split.getDeliveryTags(); + if (latestConsumedId != null) { + cursors.put(split.splitId(), latestConsumedId); + } + } + return pendingSplit; + } + + @Override + public void addSplits(List splits) { + + } + + @Override + public void handleNoMoreSplits() { + log.info("Reader received NoMoreSplits event."); + this.noMoreSplitsAssignment = true; + } + + @Override + public void notifyCheckpointComplete(long checkpointId) throws Exception { + log.debug("Committing cursors for checkpoint {}", checkpointId); + Map> pendingCursors = pendingCursorsToCommit.remove(checkpointId); + if (pendingCursors == null) { + log.debug( + "Cursors for checkpoint {} either do not exist or have already been committed.", + checkpointId); + return; + } + pendingCursors.forEach((splitId, deliveryTags) -> { + if (finishedSplits.contains(splitId)) { + return; + } + acknowledgeSessionIDs(deliveryTags); + +// if (pendingCursorsToFinish.containsKey(splitId) && +// pendingCursorsToFinish.get(splitId).compareTo(messageId) == 0) { +// finishedSplits.add(splitId); +// try { +// splitReaders.get(splitId).close(); +// } catch (IOException e) { +// throw new RuntimeException("Failed to close the split reader thread.", e); +// } +// } + }); + + } + + public boolean setMessageIdentifiers(String correlationId, long deliveryTag) { + if (customIdentifiersSet) { + throw new IllegalStateException( + "You can set only a single set of identifiers for a block of messages."); + } + + this.customIdentifiersSet = true; + if (!autoAck) { + if (usesCorrelationId) { + Preconditions.checkNotNull( + correlationId, + "RabbitMQ source was instantiated with usesCorrelationId set to " + + "true yet we couldn't extract the correlation id from it!"); + if (!addId(correlationId)) { + // we have already processed this message + try { + channel.basicReject(deliveryTag, false); + } catch (IOException e) { + throw new RuntimeException( + "Message could not be acknowledged with basicReject.", e); + } + return false; + } + } + sessionIds.add(deliveryTag); + } + return true; + } + + protected void acknowledgeSessionIDs(List sessionIds) { + try { + for (long id : sessionIds) { + channel.basicAck(id, false); + } + channel.txCommit(); + } catch (IOException e) { + throw new RuntimeException( + "Messages could not be acknowledged during checkpoint creation.", e); + } + } +} diff --git a/seatunnel-connectors-v2/connector-rabbitmq/src/main/java/org/apache/seatunnel/connectors/seatunnel/rabbitmq/source/RabbitMQSplitReaderThread.java b/seatunnel-connectors-v2/connector-rabbitmq/src/main/java/org/apache/seatunnel/connectors/seatunnel/rabbitmq/source/RabbitMQSplitReaderThread.java new file mode 100644 index 00000000000..1226afe1460 --- /dev/null +++ b/seatunnel-connectors-v2/connector-rabbitmq/src/main/java/org/apache/seatunnel/connectors/seatunnel/rabbitmq/source/RabbitMQSplitReaderThread.java @@ -0,0 +1,35 @@ +package org.apache.seatunnel.connectors.seatunnel.rabbitmq.source; + +import com.rabbitmq.client.Consumer; +import com.rabbitmq.client.Delivery; +import org.apache.seatunnel.common.Handover; +import org.apache.seatunnel.connectors.seatunnel.pulsar.source.reader.PulsarSplitReaderThread; +import org.apache.seatunnel.connectors.seatunnel.pulsar.source.reader.RecordWithSplitId; +import org.apache.seatunnel.connectors.seatunnel.rabbitmq.client.QueueingConsumer; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class RabbitMQSplitReaderThread extends Thread implements Cloneable { + private static final Logger LOG = LoggerFactory.getLogger(RabbitMQSplitReaderThread.class); + protected Consumer consumer; + protected final Handover handover; + private volatile boolean running; + + public RabbitMQSplitReaderThread(Handover handover) { + this.handover = handover; + } + + + public void open(Handover handover) { + consumer = new QueueingConsumer(null); + + } + + @Override + public void run() { + while (running) { + + } + + } +} diff --git a/seatunnel-connectors-v2/connector-rabbitmq/src/main/java/org/apache/seatunnel/connectors/seatunnel/rabbitmq/split/RabbitMQSplit.java b/seatunnel-connectors-v2/connector-rabbitmq/src/main/java/org/apache/seatunnel/connectors/seatunnel/rabbitmq/split/RabbitMQSplit.java new file mode 100644 index 00000000000..270515a1d30 --- /dev/null +++ b/seatunnel-connectors-v2/connector-rabbitmq/src/main/java/org/apache/seatunnel/connectors/seatunnel/rabbitmq/split/RabbitMQSplit.java @@ -0,0 +1,46 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.connectors.seatunnel.rabbitmq.split; + +import com.sun.istack.internal.Nullable; +import lombok.AllArgsConstructor; +import lombok.Getter; +import lombok.Setter; + +import org.apache.seatunnel.api.source.SourceSplit; +import org.apache.seatunnel.connectors.seatunnel.pulsar.source.split.PulsarPartitionSplit; + +import java.util.List; +import java.util.Set; + +@Getter +@Setter +@AllArgsConstructor +public class RabbitMQSplit implements SourceSplit { + @Nullable + private List deliveryTags; + + @Override + public String splitId() { + return ""; + } + public RabbitMQSplit copy() { + return new RabbitMQSplit(deliveryTags); + } + +} diff --git a/seatunnel-connectors-v2/connector-rabbitmq/src/main/java/org/apache/seatunnel/connectors/seatunnel/rabbitmq/split/RabbitMQState.java b/seatunnel-connectors-v2/connector-rabbitmq/src/main/java/org/apache/seatunnel/connectors/seatunnel/rabbitmq/split/RabbitMQState.java new file mode 100644 index 00000000000..b545856c406 --- /dev/null +++ b/seatunnel-connectors-v2/connector-rabbitmq/src/main/java/org/apache/seatunnel/connectors/seatunnel/rabbitmq/split/RabbitMQState.java @@ -0,0 +1,6 @@ +package org.apache.seatunnel.connectors.seatunnel.rabbitmq.split; + +import java.io.Serializable; + +public class RabbitMQState implements Serializable { +} diff --git a/seatunnel-connectors-v2/pom.xml b/seatunnel-connectors-v2/pom.xml index 37dd5223c7e..e42a1233bcf 100644 --- a/seatunnel-connectors-v2/pom.xml +++ b/seatunnel-connectors-v2/pom.xml @@ -56,6 +56,7 @@ connector-mongodb connector-iceberg connector-influxdb + connector-rabbitmq From 59721c298d73f8ff9cc11685d458e4cd29cd9fad Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=AF=95=E5=8D=9A?= Date: Fri, 28 Oct 2022 19:55:20 +0800 Subject: [PATCH 02/15] add rabbitmq connector --- .../rabbitmq/source/RabbitMQSourceReader.java | 41 ++--------- .../source/RabbitMQSplitEnumerator.java | 72 +++++++++++++++++++ .../source/RabbitMQSplitEnumeratorState.java | 26 +++++++ .../rabbitmq/split/RabbitMQSplit.java | 2 - 4 files changed, 104 insertions(+), 37 deletions(-) create mode 100644 seatunnel-connectors-v2/connector-rabbitmq/src/main/java/org/apache/seatunnel/connectors/seatunnel/rabbitmq/source/RabbitMQSplitEnumerator.java create mode 100644 seatunnel-connectors-v2/connector-rabbitmq/src/main/java/org/apache/seatunnel/connectors/seatunnel/rabbitmq/source/RabbitMQSplitEnumeratorState.java diff --git a/seatunnel-connectors-v2/connector-rabbitmq/src/main/java/org/apache/seatunnel/connectors/seatunnel/rabbitmq/source/RabbitMQSourceReader.java b/seatunnel-connectors-v2/connector-rabbitmq/src/main/java/org/apache/seatunnel/connectors/seatunnel/rabbitmq/source/RabbitMQSourceReader.java index a570f03409b..085a1099895 100644 --- a/seatunnel-connectors-v2/connector-rabbitmq/src/main/java/org/apache/seatunnel/connectors/seatunnel/rabbitmq/source/RabbitMQSourceReader.java +++ b/seatunnel-connectors-v2/connector-rabbitmq/src/main/java/org/apache/seatunnel/connectors/seatunnel/rabbitmq/source/RabbitMQSourceReader.java @@ -1,13 +1,13 @@ package org.apache.seatunnel.connectors.seatunnel.rabbitmq.source; +import com.google.common.base.Preconditions; import com.rabbitmq.client.*; import lombok.extern.slf4j.Slf4j; import org.apache.seatunnel.api.serialization.DeserializationSchema; import org.apache.seatunnel.api.source.Collector; import org.apache.seatunnel.api.source.SourceReader; import org.apache.seatunnel.common.Handover; -import org.apache.seatunnel.connectors.seatunnel.pulsar.source.split.PulsarPartitionSplit; -import org.apache.seatunnel.connectors.seatunnel.rabbitmq.split.RabbitMQSplit; + import org.apache.seatunnel.connectors.seatunnel.rabbitmq.split.RabbitMQSplit; import java.io.IOException; import java.util.*; @@ -92,11 +92,11 @@ public List snapshotState(long checkpointId) throws Exception { int size = pendingSplit.size(); Map> cursors = pendingCursorsToCommit.computeIfAbsent(checkpointId, id -> new HashMap<>(size)); - // Put the cursors of the active splits. + // Put currentCheckPoint deliveryTags. for (RabbitMQSplit split : pendingSplit) { - List latestConsumedId = split.getDeliveryTags(); - if (latestConsumedId != null) { - cursors.put(split.splitId(), latestConsumedId); + List currentCheckPointDeliveryTags = split.getDeliveryTags(); + if (currentCheckPointDeliveryTags != null) { + cursors.put(split.splitId(), currentCheckPointDeliveryTags); } } return pendingSplit; @@ -142,35 +142,6 @@ public void notifyCheckpointComplete(long checkpointId) throws Exception { } - public boolean setMessageIdentifiers(String correlationId, long deliveryTag) { - if (customIdentifiersSet) { - throw new IllegalStateException( - "You can set only a single set of identifiers for a block of messages."); - } - - this.customIdentifiersSet = true; - if (!autoAck) { - if (usesCorrelationId) { - Preconditions.checkNotNull( - correlationId, - "RabbitMQ source was instantiated with usesCorrelationId set to " - + "true yet we couldn't extract the correlation id from it!"); - if (!addId(correlationId)) { - // we have already processed this message - try { - channel.basicReject(deliveryTag, false); - } catch (IOException e) { - throw new RuntimeException( - "Message could not be acknowledged with basicReject.", e); - } - return false; - } - } - sessionIds.add(deliveryTag); - } - return true; - } - protected void acknowledgeSessionIDs(List sessionIds) { try { for (long id : sessionIds) { diff --git a/seatunnel-connectors-v2/connector-rabbitmq/src/main/java/org/apache/seatunnel/connectors/seatunnel/rabbitmq/source/RabbitMQSplitEnumerator.java b/seatunnel-connectors-v2/connector-rabbitmq/src/main/java/org/apache/seatunnel/connectors/seatunnel/rabbitmq/source/RabbitMQSplitEnumerator.java new file mode 100644 index 00000000000..89d88b0d18c --- /dev/null +++ b/seatunnel-connectors-v2/connector-rabbitmq/src/main/java/org/apache/seatunnel/connectors/seatunnel/rabbitmq/source/RabbitMQSplitEnumerator.java @@ -0,0 +1,72 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.connectors.seatunnel.rabbitmq.source; + +import org.apache.seatunnel.api.source.SourceSplitEnumerator; + +import java.io.IOException; +import java.util.List; + + +public class RabbitMQSplitEnumerator implements SourceSplitEnumerator { + + @Override + public void open() { + + } + + @Override + public void run() throws Exception { + + } + + @Override + public void close() throws IOException { + + } + + @Override + public void addSplitsBack(List splits, int subtaskId) { + + } + + @Override + public int currentUnassignedSplitSize() { + return 0; + } + + @Override + public void handleSplitRequest(int subtaskId) { + + } + + @Override + public void registerReader(int subtaskId) { + + } + + @Override + public Object snapshotState(long checkpointId) throws Exception { + return null; + } + + @Override + public void notifyCheckpointComplete(long checkpointId) throws Exception { + + } +} diff --git a/seatunnel-connectors-v2/connector-rabbitmq/src/main/java/org/apache/seatunnel/connectors/seatunnel/rabbitmq/source/RabbitMQSplitEnumeratorState.java b/seatunnel-connectors-v2/connector-rabbitmq/src/main/java/org/apache/seatunnel/connectors/seatunnel/rabbitmq/source/RabbitMQSplitEnumeratorState.java new file mode 100644 index 00000000000..988fe3f74bd --- /dev/null +++ b/seatunnel-connectors-v2/connector-rabbitmq/src/main/java/org/apache/seatunnel/connectors/seatunnel/rabbitmq/source/RabbitMQSplitEnumeratorState.java @@ -0,0 +1,26 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.connectors.seatunnel.rabbitmq.source; + + +import java.io.Serializable; + +public class RabbitMQSplitEnumeratorState implements Serializable { + +} diff --git a/seatunnel-connectors-v2/connector-rabbitmq/src/main/java/org/apache/seatunnel/connectors/seatunnel/rabbitmq/split/RabbitMQSplit.java b/seatunnel-connectors-v2/connector-rabbitmq/src/main/java/org/apache/seatunnel/connectors/seatunnel/rabbitmq/split/RabbitMQSplit.java index 270515a1d30..6de8eeaa04c 100644 --- a/seatunnel-connectors-v2/connector-rabbitmq/src/main/java/org/apache/seatunnel/connectors/seatunnel/rabbitmq/split/RabbitMQSplit.java +++ b/seatunnel-connectors-v2/connector-rabbitmq/src/main/java/org/apache/seatunnel/connectors/seatunnel/rabbitmq/split/RabbitMQSplit.java @@ -23,10 +23,8 @@ import lombok.Setter; import org.apache.seatunnel.api.source.SourceSplit; -import org.apache.seatunnel.connectors.seatunnel.pulsar.source.split.PulsarPartitionSplit; import java.util.List; -import java.util.Set; @Getter @Setter From 837a0b4bae2a638643dd9ad2777d96ca62dffa4e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=AF=95=E5=8D=9A?= Date: Mon, 31 Oct 2022 10:12:31 +0800 Subject: [PATCH 03/15] [Feature][Connector-V2] add rabbitmq connector --- plugin-mapping.properties | 3 +- .../connector-rabbitmq/pom.xml | 10 + .../rabbitmq/client/QueueingConsumer.java | 63 ++++-- .../rabbitmq/client/RabbitmqClient.java | 197 +++++++++++++++++ .../rabbitmq/config/RabbitmqConfig.java | 120 ++++++++++ .../seatunnel/rabbitmq/sink/RabbitmqSink.java | 67 ++++++ .../rabbitmq/sink/RabbitmqSinkWriter.java | 59 +++++ .../rabbitmq/source/DeliveryMessage.java | 31 +++ .../rabbitmq/source/DeliveryWithSplitId.java | 14 -- .../rabbitmq/source/RabbitMQSourceReader.java | 156 ------------- .../source/RabbitMQSplitEnumeratorState.java | 26 --- .../source/RabbitMQSplitReaderThread.java | 35 --- ...abbitMQSource.java => RabbitmqSource.java} | 44 ++-- .../rabbitmq/source/RabbitmqSourceReader.java | 205 ++++++++++++++++++ ...ator.java => RabbitmqSplitEnumerator.java} | 5 +- .../rabbitmq/split/RabbitMQState.java | 6 - ...{RabbitMQSplit.java => RabbitmqSplit.java} | 14 +- .../split/RabbitmqSplitEnumeratorState.java | 23 ++ .../connector-rabbitmq-e2e/pom.xml | 43 ++++ .../e2e/connector/rabbitmq/RabbitmqIT.java | 203 +++++++++++++++++ .../src/test/resources/log4j.properties | 22 ++ .../test/resources/rabbitmq-to-rabbitmq.conf | 59 +++++ .../seatunnel-connector-v2-e2e/pom.xml | 1 + 23 files changed, 1121 insertions(+), 285 deletions(-) create mode 100644 seatunnel-connectors-v2/connector-rabbitmq/src/main/java/org/apache/seatunnel/connectors/seatunnel/rabbitmq/client/RabbitmqClient.java create mode 100644 seatunnel-connectors-v2/connector-rabbitmq/src/main/java/org/apache/seatunnel/connectors/seatunnel/rabbitmq/config/RabbitmqConfig.java create mode 100644 seatunnel-connectors-v2/connector-rabbitmq/src/main/java/org/apache/seatunnel/connectors/seatunnel/rabbitmq/sink/RabbitmqSink.java create mode 100644 seatunnel-connectors-v2/connector-rabbitmq/src/main/java/org/apache/seatunnel/connectors/seatunnel/rabbitmq/sink/RabbitmqSinkWriter.java create mode 100644 seatunnel-connectors-v2/connector-rabbitmq/src/main/java/org/apache/seatunnel/connectors/seatunnel/rabbitmq/source/DeliveryMessage.java delete mode 100644 seatunnel-connectors-v2/connector-rabbitmq/src/main/java/org/apache/seatunnel/connectors/seatunnel/rabbitmq/source/DeliveryWithSplitId.java delete mode 100644 seatunnel-connectors-v2/connector-rabbitmq/src/main/java/org/apache/seatunnel/connectors/seatunnel/rabbitmq/source/RabbitMQSourceReader.java delete mode 100644 seatunnel-connectors-v2/connector-rabbitmq/src/main/java/org/apache/seatunnel/connectors/seatunnel/rabbitmq/source/RabbitMQSplitEnumeratorState.java delete mode 100644 seatunnel-connectors-v2/connector-rabbitmq/src/main/java/org/apache/seatunnel/connectors/seatunnel/rabbitmq/source/RabbitMQSplitReaderThread.java rename seatunnel-connectors-v2/connector-rabbitmq/src/main/java/org/apache/seatunnel/connectors/seatunnel/rabbitmq/source/{RabbitMQSource.java => RabbitmqSource.java} (58%) create mode 100644 seatunnel-connectors-v2/connector-rabbitmq/src/main/java/org/apache/seatunnel/connectors/seatunnel/rabbitmq/source/RabbitmqSourceReader.java rename seatunnel-connectors-v2/connector-rabbitmq/src/main/java/org/apache/seatunnel/connectors/seatunnel/rabbitmq/source/{RabbitMQSplitEnumerator.java => RabbitmqSplitEnumerator.java} (95%) delete mode 100644 seatunnel-connectors-v2/connector-rabbitmq/src/main/java/org/apache/seatunnel/connectors/seatunnel/rabbitmq/split/RabbitMQState.java rename seatunnel-connectors-v2/connector-rabbitmq/src/main/java/org/apache/seatunnel/connectors/seatunnel/rabbitmq/split/{RabbitMQSplit.java => RabbitmqSplit.java} (85%) create mode 100644 seatunnel-connectors-v2/connector-rabbitmq/src/main/java/org/apache/seatunnel/connectors/seatunnel/rabbitmq/split/RabbitmqSplitEnumeratorState.java create mode 100644 seatunnel-e2e/seatunnel-connector-v2-e2e/connector-rabbitmq-e2e/pom.xml create mode 100644 seatunnel-e2e/seatunnel-connector-v2-e2e/connector-rabbitmq-e2e/src/test/java/org/apache/seatunnel/e2e/connector/rabbitmq/RabbitmqIT.java create mode 100644 seatunnel-e2e/seatunnel-connector-v2-e2e/connector-rabbitmq-e2e/src/test/resources/log4j.properties create mode 100644 seatunnel-e2e/seatunnel-connector-v2-e2e/connector-rabbitmq-e2e/src/test/resources/rabbitmq-to-rabbitmq.conf diff --git a/plugin-mapping.properties b/plugin-mapping.properties index e022fa59b1b..7840373db81 100644 --- a/plugin-mapping.properties +++ b/plugin-mapping.properties @@ -135,4 +135,5 @@ seatunnel.sink.MongoDB = connector-mongodb seatunnel.source.Iceberg = connector-iceberg seatunnel.source.InfluxDB = connector-influxdb seatunnel.source.S3File = connector-file-s3 -seatunnel.sink.S3File = connector-file-s3 \ No newline at end of file +seatunnel.sink.S3File = connector-file-s3 +seatunnel.source.RabbitMQ = connector-rabbitmq \ No newline at end of file diff --git a/seatunnel-connectors-v2/connector-rabbitmq/pom.xml b/seatunnel-connectors-v2/connector-rabbitmq/pom.xml index 43ff333f20e..d10ea57ba9a 100644 --- a/seatunnel-connectors-v2/connector-rabbitmq/pom.xml +++ b/seatunnel-connectors-v2/connector-rabbitmq/pom.xml @@ -43,5 +43,15 @@ amqp-client ${rabbitmq.version} + + org.apache.seatunnel + seatunnel-format-json + ${project.version} + + + org.apache.seatunnel + seatunnel-format-text + ${project.version} + diff --git a/seatunnel-connectors-v2/connector-rabbitmq/src/main/java/org/apache/seatunnel/connectors/seatunnel/rabbitmq/client/QueueingConsumer.java b/seatunnel-connectors-v2/connector-rabbitmq/src/main/java/org/apache/seatunnel/connectors/seatunnel/rabbitmq/client/QueueingConsumer.java index b9bf30bf95f..c77e81fce26 100644 --- a/seatunnel-connectors-v2/connector-rabbitmq/src/main/java/org/apache/seatunnel/connectors/seatunnel/rabbitmq/client/QueueingConsumer.java +++ b/seatunnel-connectors-v2/connector-rabbitmq/src/main/java/org/apache/seatunnel/connectors/seatunnel/rabbitmq/client/QueueingConsumer.java @@ -1,5 +1,24 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package org.apache.seatunnel.connectors.seatunnel.rabbitmq.client; +import org.apache.seatunnel.common.Handover; + import com.rabbitmq.client.AMQP; import com.rabbitmq.client.Channel; import com.rabbitmq.client.ConsumerCancelledException; @@ -8,14 +27,15 @@ import com.rabbitmq.client.Envelope; import com.rabbitmq.client.ShutdownSignalException; import com.rabbitmq.utility.Utility; +import lombok.SneakyThrows; +import lombok.extern.slf4j.Slf4j; import java.io.IOException; -import java.util.concurrent.BlockingQueue; -import java.util.concurrent.LinkedBlockingQueue; import java.util.concurrent.TimeUnit; +@Slf4j public class QueueingConsumer extends DefaultConsumer { - private final BlockingQueue queue; + private final Handover handover; // When this is non-null the queue is in shutdown mode and nextDelivery should // throw a shutdown signal exception. @@ -24,13 +44,13 @@ public class QueueingConsumer extends DefaultConsumer { private static final Delivery POISON = new Delivery(null, null, null); - public QueueingConsumer(Channel channel) { - this(channel, Integer.MAX_VALUE); + public QueueingConsumer(Channel channel, Handover handover) { + this(channel, Integer.MAX_VALUE, handover); } - public QueueingConsumer(Channel channel, int capacity) { + public QueueingConsumer(Channel channel, int capacity, Handover handover) { super(channel); - this.queue = new LinkedBlockingQueue<>(capacity); + this.handover = handover; } private void checkShutdown() { @@ -39,10 +59,10 @@ private void checkShutdown() { } } - private Delivery handle(Delivery delivery) { + private Delivery handle(Delivery delivery) throws Handover.ClosedException, InterruptedException { if (delivery == POISON || delivery == null && (shutdown != null || cancelled != null)) { if (delivery == POISON) { - queue.add(POISON); + handover.produce(POISON); if (shutdown == null && cancelled == null) { throw new IllegalStateException( "POISON in queue, but null shutdown and null cancelled. " @@ -60,37 +80,46 @@ private Delivery handle(Delivery delivery) { } public Delivery nextDelivery() - throws InterruptedException, ShutdownSignalException, ConsumerCancelledException { - return handle(queue.take()); + throws Exception { + return handle(handover.pollNext().get()); } public Delivery nextDelivery(long timeout) - throws InterruptedException, ShutdownSignalException, ConsumerCancelledException { + throws Exception { return nextDelivery(timeout, TimeUnit.MILLISECONDS); } public Delivery nextDelivery(long timeout, TimeUnit unit) - throws InterruptedException, ShutdownSignalException, ConsumerCancelledException { - return handle(queue.poll(timeout, unit)); + throws Exception { + return handle(handover.pollNext().get()); } @Override public void handleShutdownSignal(String consumerTag, ShutdownSignalException sig) { shutdown = sig; - queue.add(POISON); + try { + handover.produce(POISON); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } catch (Handover.ClosedException e) { + throw new RuntimeException(e); + } } + @SneakyThrows @Override public void handleCancel(String consumerTag) throws IOException { cancelled = new ConsumerCancelledException(); - queue.add(POISON); + handover.produce(POISON); } + @SneakyThrows @Override public void handleDelivery( String consumerTag, Envelope envelope, AMQP.BasicProperties properties, byte[] body) throws IOException { checkShutdown(); - this.queue.add(new Delivery(envelope, properties, body)); + log.info(new String(body) + "?????????"); + handover.produce(new Delivery(envelope, properties, body)); } } diff --git a/seatunnel-connectors-v2/connector-rabbitmq/src/main/java/org/apache/seatunnel/connectors/seatunnel/rabbitmq/client/RabbitmqClient.java b/seatunnel-connectors-v2/connector-rabbitmq/src/main/java/org/apache/seatunnel/connectors/seatunnel/rabbitmq/client/RabbitmqClient.java new file mode 100644 index 00000000000..4896f7ec1e3 --- /dev/null +++ b/seatunnel-connectors-v2/connector-rabbitmq/src/main/java/org/apache/seatunnel/connectors/seatunnel/rabbitmq/client/RabbitmqClient.java @@ -0,0 +1,197 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.connectors.seatunnel.rabbitmq.client; + +import org.apache.seatunnel.common.Handover; +import org.apache.seatunnel.connectors.seatunnel.rabbitmq.config.RabbitmqConfig; + +import com.rabbitmq.client.Channel; +import com.rabbitmq.client.Connection; +import com.rabbitmq.client.ConnectionFactory; +import com.rabbitmq.client.DefaultConsumer; +import com.rabbitmq.client.Delivery; +import lombok.AllArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.StringUtils; + +import java.io.IOException; +import java.net.URISyntaxException; +import java.security.KeyManagementException; +import java.security.NoSuchAlgorithmException; +import java.util.concurrent.TimeoutException; + +@Slf4j +@AllArgsConstructor +public class RabbitmqClient { + private RabbitmqConfig config; + private ConnectionFactory connectionFactory; + private Connection connection; + private Channel channel; + + public RabbitmqClient(RabbitmqConfig config) { + this.config = config; + try { + this.connectionFactory = getConnectionFactory(); + this.connection = connectionFactory.newConnection(); + this.channel = connection.createChannel(); + setupQueue(); + } catch (Exception e) { + throw new RuntimeException( + "Error while create RMQ client with " + + this.config.getQueueName() + + " at " + + this.config.getHost(), + e); + } + + } + + public Channel getChannel() { + return channel; + } + + public DefaultConsumer getQueueingConsumer(Handover handover) { + DefaultConsumer consumer = new QueueingConsumer(channel, handover); + return consumer; + } + + public ConnectionFactory getConnectionFactory() + throws URISyntaxException, NoSuchAlgorithmException, KeyManagementException { + ConnectionFactory factory = new ConnectionFactory(); + if (!StringUtils.isEmpty(config.getUri())) { + try { + factory.setUri(config.getUri()); + } catch (URISyntaxException e) { + log.error("Failed to parse uri", e); + throw e; + } catch (KeyManagementException e) { + // this should never happen + log.error("Failed to initialize ssl context.", e); + throw e; + } catch (NoSuchAlgorithmException e) { + // this should never happen + log.error("Failed to setup ssl factory.", e); + throw e; + } + } else { + factory.setHost(config.getHost()); + factory.setPort(config.getPort()); + factory.setVirtualHost(config.getVirtualHost()); + factory.setUsername(config.getUsername()); + factory.setPassword(config.getPassword()); + } + + if (config.getAutomaticRecovery() != null) { + factory.setAutomaticRecoveryEnabled(config.getAutomaticRecovery()); + } + if (config.getConnectionTimeout() != null) { + factory.setConnectionTimeout(config.getConnectionTimeout()); + } + if (config.getNetworkRecoveryInterval() != null) { + factory.setNetworkRecoveryInterval(config.getNetworkRecoveryInterval()); + } + if (config.getRequestedHeartbeat() != null) { + factory.setRequestedHeartbeat(config.getRequestedHeartbeat()); + } + if (config.getTopologyRecovery() != null) { + factory.setTopologyRecoveryEnabled(config.getTopologyRecovery()); + } + if (config.getRequestedChannelMax() != null) { + factory.setRequestedChannelMax(config.getRequestedChannelMax()); + } + if (config.getRequestedFrameMax() != null) { + factory.setRequestedFrameMax(config.getRequestedFrameMax()); + } + return factory; + } + + public void write(byte[] msg) { + try { + if (StringUtils.isEmpty(config.getRoutingKey())) { + channel.basicPublish("", config.getQueueName(), null, msg); + } else { + //not support set returnListener + channel.basicPublish( + config.getExchange(), + config.getRoutingKey(), + false, + false, + null, + msg); + } + } catch (IOException e) { + if (config.isLogFailuresOnly()) { + log.error( + "Cannot send RMQ message {} at {}", + config.getQueueName(), + config.getHost(), + e); + } else { + throw new RuntimeException( + "Cannot send RMQ message " + + config.getQueueName() + + " at " + + config.getHost(), + e); + } + } + } + + public void close() { + Exception t = null; + try { + if (channel != null) { + channel.close(); + } + } catch (IOException | TimeoutException e) { + t = e; + } + + try { + if (connection != null) { + connection.close(); + } + } catch (IOException e) { + if (t != null) { + log.warn( + "Both channel and connection closing failed. Logging channel exception and failing with connection exception", + t); + } + t = e; + } + if (t != null) { + throw new RuntimeException( + "Error while closing RMQ connection with " + + this.config.getQueueName() + + " at " + + this.config.getHost(), + t); + } + } + + protected void setupQueue() throws IOException { + if (config.getQueueName() != null) { + declareQueueDefaults(channel, config.getQueueName()); + } + } + + private void declareQueueDefaults(Channel channel, String queueName) throws IOException { + channel.queueDeclare(queueName, true, false, false, null); + } + +} diff --git a/seatunnel-connectors-v2/connector-rabbitmq/src/main/java/org/apache/seatunnel/connectors/seatunnel/rabbitmq/config/RabbitmqConfig.java b/seatunnel-connectors-v2/connector-rabbitmq/src/main/java/org/apache/seatunnel/connectors/seatunnel/rabbitmq/config/RabbitmqConfig.java new file mode 100644 index 00000000000..3192d52612d --- /dev/null +++ b/seatunnel-connectors-v2/connector-rabbitmq/src/main/java/org/apache/seatunnel/connectors/seatunnel/rabbitmq/config/RabbitmqConfig.java @@ -0,0 +1,120 @@ +package org.apache.seatunnel.connectors.seatunnel.rabbitmq.config; + +import org.apache.seatunnel.common.config.TypesafeConfigUtils; + +import org.apache.seatunnel.shade.com.typesafe.config.Config; + +import lombok.AllArgsConstructor; +import lombok.Getter; +import lombok.Setter; + +import java.io.Serializable; +import java.util.HashMap; +import java.util.Map; + +@Setter +@Getter +@AllArgsConstructor +public class RabbitmqConfig implements Serializable { + public static final String HOST = "host"; + public static final String PORT = "port"; + public static final String VIRTUAL_HOST = "virtual_host"; + public static final String USERNAME = "username"; + public static final String PASSWORD = "password"; + public static final String URL = "url"; + public static final String NETWORK_RECOVERY_INTERVAL = "network_recovery_interval"; + public static final String AUTOMATIC_RECOVERY = "automatic_recovery"; + public static final String TOPOLOGY_RECOVERY = "topology_recovery"; + public static final String CONNECTION_TIMEOUT = "connection_timeout"; + public static final String REQUESTED_CHANNEL_MAX = "requested_channel_max"; + public static final String REQUESTED_FRAME_MAX = "requested_frame_max"; + public static final String REQUESTED_HEARTBEAT = "requested_heartbeat"; + + public static final String PREFETCH_COUNT = "prefetch_count"; + public static final String DELIVERY_TIMEOUT = "delivery_timeout"; + public static final String QUEUE_NAME = "queue_name"; + public static final String ROUTING_KEY = "routing_key"; + public static final String EXCHANGE = "exchange"; + + + public static final String LOG_FAILURES_ONLY = "log_failures_only"; + + private String host; + private Integer port; + private String virtualHost; + private String username; + private String password; + private String uri; + private Integer networkRecoveryInterval; + private Boolean automaticRecovery; + private Boolean topologyRecovery; + private Integer connectionTimeout; + private Integer requestedChannelMax; + private Integer requestedFrameMax; + private Integer requestedHeartbeat; + private Integer prefetchCount; + private long deliveryTimeout; + private String queueName; + private String routingKey; + private boolean logFailuresOnly = false; + private String exchange = ""; + public static final String RABBITMQ_SINK_CONFIG_PREFIX = "rabbitmq.properties."; + + private final Map sinkOptionProps = new HashMap<>(); + + private void parseSinkOptionProperties(Config pluginConfig) { + Config sinkOptionConfig = TypesafeConfigUtils.extractSubConfig(pluginConfig, + RABBITMQ_SINK_CONFIG_PREFIX, false); + sinkOptionConfig.entrySet().forEach(entry -> { + final String configKey = entry.getKey().toLowerCase(); + this.sinkOptionProps.put(configKey, entry.getValue().unwrapped()); + }); + } + + public RabbitmqConfig(Config config) { + this.host = config.getString(HOST); + this.port = config.getInt(PORT); + this.queueName = config.getString(QUEUE_NAME); + if (config.hasPath(USERNAME)) { + this.username = config.getString(USERNAME); + } + if (config.hasPath(PASSWORD)) { + this.password = config.getString(PASSWORD); + } + if (config.hasPath(VIRTUAL_HOST)) { + this.virtualHost = config.getString(VIRTUAL_HOST); + } + if (config.hasPath(NETWORK_RECOVERY_INTERVAL)) { + this.networkRecoveryInterval = config.getInt(NETWORK_RECOVERY_INTERVAL); + } + if (config.hasPath(AUTOMATIC_RECOVERY)) { + this.automaticRecovery = config.getBoolean(AUTOMATIC_RECOVERY); + } + if (config.hasPath(CONNECTION_TIMEOUT)) { + this.connectionTimeout = config.getInt(CONNECTION_TIMEOUT); + } + if (config.hasPath(REQUESTED_CHANNEL_MAX)) { + this.requestedChannelMax = config.getInt(REQUESTED_CHANNEL_MAX); + } + if (config.hasPath(REQUESTED_FRAME_MAX)) { + this.requestedFrameMax = config.getInt(REQUESTED_FRAME_MAX); + } + if (config.hasPath(REQUESTED_HEARTBEAT)) { + this.requestedHeartbeat = config.getInt(REQUESTED_HEARTBEAT); + } + if (config.hasPath(PREFETCH_COUNT)) { + this.prefetchCount = config.getInt(PREFETCH_COUNT); + } + if (config.hasPath(DELIVERY_TIMEOUT)) { + this.deliveryTimeout = config.getInt(DELIVERY_TIMEOUT); + } + if (config.hasPath(ROUTING_KEY)) { + this.routingKey = config.getString(ROUTING_KEY); + } + if (config.hasPath(EXCHANGE)) { + this.exchange = config.getString(EXCHANGE); + } + parseSinkOptionProperties(config); + + } +} diff --git a/seatunnel-connectors-v2/connector-rabbitmq/src/main/java/org/apache/seatunnel/connectors/seatunnel/rabbitmq/sink/RabbitmqSink.java b/seatunnel-connectors-v2/connector-rabbitmq/src/main/java/org/apache/seatunnel/connectors/seatunnel/rabbitmq/sink/RabbitmqSink.java new file mode 100644 index 00000000000..66af5ea5339 --- /dev/null +++ b/seatunnel-connectors-v2/connector-rabbitmq/src/main/java/org/apache/seatunnel/connectors/seatunnel/rabbitmq/sink/RabbitmqSink.java @@ -0,0 +1,67 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.connectors.seatunnel.rabbitmq.sink; + +import org.apache.seatunnel.api.common.PrepareFailException; +import org.apache.seatunnel.api.sink.SeaTunnelSink; +import org.apache.seatunnel.api.sink.SinkWriter; +import org.apache.seatunnel.api.table.type.SeaTunnelDataType; +import org.apache.seatunnel.api.table.type.SeaTunnelRow; +import org.apache.seatunnel.api.table.type.SeaTunnelRowType; +import org.apache.seatunnel.connectors.seatunnel.common.sink.AbstractSimpleSink; +import org.apache.seatunnel.connectors.seatunnel.common.sink.AbstractSinkWriter; +import org.apache.seatunnel.connectors.seatunnel.rabbitmq.config.RabbitmqConfig; + +import org.apache.seatunnel.shade.com.typesafe.config.Config; + +import com.google.auto.service.AutoService; + +import java.io.IOException; + +@AutoService(SeaTunnelSink.class) +public class RabbitmqSink extends AbstractSimpleSink { + private SeaTunnelRowType seaTunnelRowType; + private Config pluginConfig; + private RabbitmqConfig rabbitMQConfig; + + @Override + public String getPluginName() { + return "RabbitMQ"; + } + + @Override + public void prepare(Config pluginConfig) throws PrepareFailException { + this.pluginConfig = pluginConfig; + rabbitMQConfig = new RabbitmqConfig(pluginConfig); + } + + @Override + public void setTypeInfo(SeaTunnelRowType seaTunnelRowType) { + this.seaTunnelRowType = seaTunnelRowType; + } + + @Override + public SeaTunnelDataType getConsumedType() { + return seaTunnelRowType; + } + + @Override + public AbstractSinkWriter createWriter(SinkWriter.Context context) throws IOException { + return new RabbitmqSinkWriter(rabbitMQConfig, seaTunnelRowType); + } +} diff --git a/seatunnel-connectors-v2/connector-rabbitmq/src/main/java/org/apache/seatunnel/connectors/seatunnel/rabbitmq/sink/RabbitmqSinkWriter.java b/seatunnel-connectors-v2/connector-rabbitmq/src/main/java/org/apache/seatunnel/connectors/seatunnel/rabbitmq/sink/RabbitmqSinkWriter.java new file mode 100644 index 00000000000..5be04b4d3ba --- /dev/null +++ b/seatunnel-connectors-v2/connector-rabbitmq/src/main/java/org/apache/seatunnel/connectors/seatunnel/rabbitmq/sink/RabbitmqSinkWriter.java @@ -0,0 +1,59 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.connectors.seatunnel.rabbitmq.sink; + +import org.apache.seatunnel.api.table.type.SeaTunnelRow; +import org.apache.seatunnel.api.table.type.SeaTunnelRowType; +import org.apache.seatunnel.connectors.seatunnel.common.sink.AbstractSinkWriter; +import org.apache.seatunnel.connectors.seatunnel.rabbitmq.client.RabbitmqClient; +import org.apache.seatunnel.connectors.seatunnel.rabbitmq.config.RabbitmqConfig; +import org.apache.seatunnel.format.json.JsonSerializationSchema; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.Optional; + +public class RabbitmqSinkWriter extends AbstractSinkWriter { + private static final Logger LOG = LoggerFactory.getLogger(RabbitmqSinkWriter.class); + private RabbitmqClient rabbitMQClient; + private final JsonSerializationSchema jsonSerializationSchema; + + public RabbitmqSinkWriter(RabbitmqConfig config, SeaTunnelRowType seaTunnelRowType) { + this.rabbitMQClient = new RabbitmqClient(config); + this.jsonSerializationSchema = new JsonSerializationSchema(seaTunnelRowType); + } + + @Override + public void write(SeaTunnelRow element) { + rabbitMQClient.write(jsonSerializationSchema.serialize(element)); + } + + @Override + public Optional prepareCommit() { + + return Optional.empty(); + } + + @Override + public void close() { + if (rabbitMQClient != null) { + rabbitMQClient.close(); + } + } +} diff --git a/seatunnel-connectors-v2/connector-rabbitmq/src/main/java/org/apache/seatunnel/connectors/seatunnel/rabbitmq/source/DeliveryMessage.java b/seatunnel-connectors-v2/connector-rabbitmq/src/main/java/org/apache/seatunnel/connectors/seatunnel/rabbitmq/source/DeliveryMessage.java new file mode 100644 index 00000000000..e3589dd35d5 --- /dev/null +++ b/seatunnel-connectors-v2/connector-rabbitmq/src/main/java/org/apache/seatunnel/connectors/seatunnel/rabbitmq/source/DeliveryMessage.java @@ -0,0 +1,31 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.connectors.seatunnel.rabbitmq.source; + +import com.rabbitmq.client.Delivery; +import lombok.AllArgsConstructor; +import lombok.Getter; +import lombok.Setter; + +@AllArgsConstructor +@Setter +@Getter +public final class DeliveryMessage { + private final Delivery delivery; + private final String splitId; +} diff --git a/seatunnel-connectors-v2/connector-rabbitmq/src/main/java/org/apache/seatunnel/connectors/seatunnel/rabbitmq/source/DeliveryWithSplitId.java b/seatunnel-connectors-v2/connector-rabbitmq/src/main/java/org/apache/seatunnel/connectors/seatunnel/rabbitmq/source/DeliveryWithSplitId.java deleted file mode 100644 index 0e7dc4ade22..00000000000 --- a/seatunnel-connectors-v2/connector-rabbitmq/src/main/java/org/apache/seatunnel/connectors/seatunnel/rabbitmq/source/DeliveryWithSplitId.java +++ /dev/null @@ -1,14 +0,0 @@ -package org.apache.seatunnel.connectors.seatunnel.rabbitmq.source; - -import com.rabbitmq.client.Delivery; -import lombok.AllArgsConstructor; -import lombok.Getter; -import lombok.Setter; - -@AllArgsConstructor -@Setter -@Getter -public final class DeliveryWithSplitId { - private final Delivery delivery; - private final String splitId; -} diff --git a/seatunnel-connectors-v2/connector-rabbitmq/src/main/java/org/apache/seatunnel/connectors/seatunnel/rabbitmq/source/RabbitMQSourceReader.java b/seatunnel-connectors-v2/connector-rabbitmq/src/main/java/org/apache/seatunnel/connectors/seatunnel/rabbitmq/source/RabbitMQSourceReader.java deleted file mode 100644 index 085a1099895..00000000000 --- a/seatunnel-connectors-v2/connector-rabbitmq/src/main/java/org/apache/seatunnel/connectors/seatunnel/rabbitmq/source/RabbitMQSourceReader.java +++ /dev/null @@ -1,156 +0,0 @@ -package org.apache.seatunnel.connectors.seatunnel.rabbitmq.source; - -import com.google.common.base.Preconditions; -import com.rabbitmq.client.*; -import lombok.extern.slf4j.Slf4j; -import org.apache.seatunnel.api.serialization.DeserializationSchema; -import org.apache.seatunnel.api.source.Collector; -import org.apache.seatunnel.api.source.SourceReader; -import org.apache.seatunnel.common.Handover; - import org.apache.seatunnel.connectors.seatunnel.rabbitmq.split.RabbitMQSplit; - -import java.io.IOException; -import java.util.*; -import java.util.stream.Collectors; - -@Slf4j -public class RabbitMQSourceReader implements SourceReader { - protected final Handover handover; - protected final DeserializationSchema deserialization; - - private boolean noMoreSplitsAssignment = false; - - protected final SourceReader.Context context; - protected final int batchSize; - - protected transient Connection connection; - protected transient Channel channel; - private final boolean usesCorrelationId; - protected transient boolean autoAck; - - private transient volatile boolean running; - - protected transient List sessionIds; - protected final Map splitStates; - protected final Set finishedSplits; - protected final SortedMap>> pendingCursorsToCommit; - protected final Map> pendingCursorsToFinish; - - public RabbitMQSourceReader(Handover handover, - DeserializationSchema deserialization, - SourceReader.Context context, - int batchSize, - boolean usesCorrelationId) { - this.handover = new Handover<>(); - this.deserialization = deserialization; - this.pendingCursorsToCommit = Collections.synchronizedSortedMap(new TreeMap<>()); - this.pendingCursorsToFinish = Collections.synchronizedSortedMap(new TreeMap<>()); - this.finishedSplits = new TreeSet<>(); - this.splitStates = new HashMap<>(); - this.context = context; - this.batchSize = batchSize; - this.usesCorrelationId = usesCorrelationId; - } - - @Override - public void open() throws Exception { - sessionIds = new ArrayList<>(64); - } - - @Override - public void close() throws IOException { - - } - - @Override - public void pollNext(Collector output) throws Exception { - for (int i = 0; i < batchSize; i++) { - Optional deliveryOptional = handover.pollNext(); - if (deliveryOptional.isPresent()) { - Delivery delivery = deliveryOptional.get().getDelivery(); - AMQP.BasicProperties properties = delivery.getProperties(); - byte[] body = delivery.getBody(); - Envelope envelope = delivery.getEnvelope(); - synchronized (output.getCheckpointLock()) { - splitStates.get(deliveryOptional.get().getSplitId()).getDeliveryTags().add(envelope.getDeliveryTag()); - deserialization.deserialize(body, output); - } - } - if (noMoreSplitsAssignment && finishedSplits.size() == splitStates.size()) { - context.signalNoMoreElement(); - break; - } - } - } - - @Override - public List snapshotState(long checkpointId) throws Exception { - List pendingSplit = splitStates.values().stream() - .map(RabbitMQSplit::copy) - .collect(Collectors.toList()); - // Perform a snapshot for these splits. - int size = pendingSplit.size(); - Map> cursors = - pendingCursorsToCommit.computeIfAbsent(checkpointId, id -> new HashMap<>(size)); - // Put currentCheckPoint deliveryTags. - for (RabbitMQSplit split : pendingSplit) { - List currentCheckPointDeliveryTags = split.getDeliveryTags(); - if (currentCheckPointDeliveryTags != null) { - cursors.put(split.splitId(), currentCheckPointDeliveryTags); - } - } - return pendingSplit; - } - - @Override - public void addSplits(List splits) { - - } - - @Override - public void handleNoMoreSplits() { - log.info("Reader received NoMoreSplits event."); - this.noMoreSplitsAssignment = true; - } - - @Override - public void notifyCheckpointComplete(long checkpointId) throws Exception { - log.debug("Committing cursors for checkpoint {}", checkpointId); - Map> pendingCursors = pendingCursorsToCommit.remove(checkpointId); - if (pendingCursors == null) { - log.debug( - "Cursors for checkpoint {} either do not exist or have already been committed.", - checkpointId); - return; - } - pendingCursors.forEach((splitId, deliveryTags) -> { - if (finishedSplits.contains(splitId)) { - return; - } - acknowledgeSessionIDs(deliveryTags); - -// if (pendingCursorsToFinish.containsKey(splitId) && -// pendingCursorsToFinish.get(splitId).compareTo(messageId) == 0) { -// finishedSplits.add(splitId); -// try { -// splitReaders.get(splitId).close(); -// } catch (IOException e) { -// throw new RuntimeException("Failed to close the split reader thread.", e); -// } -// } - }); - - } - - protected void acknowledgeSessionIDs(List sessionIds) { - try { - for (long id : sessionIds) { - channel.basicAck(id, false); - } - channel.txCommit(); - } catch (IOException e) { - throw new RuntimeException( - "Messages could not be acknowledged during checkpoint creation.", e); - } - } -} diff --git a/seatunnel-connectors-v2/connector-rabbitmq/src/main/java/org/apache/seatunnel/connectors/seatunnel/rabbitmq/source/RabbitMQSplitEnumeratorState.java b/seatunnel-connectors-v2/connector-rabbitmq/src/main/java/org/apache/seatunnel/connectors/seatunnel/rabbitmq/source/RabbitMQSplitEnumeratorState.java deleted file mode 100644 index 988fe3f74bd..00000000000 --- a/seatunnel-connectors-v2/connector-rabbitmq/src/main/java/org/apache/seatunnel/connectors/seatunnel/rabbitmq/source/RabbitMQSplitEnumeratorState.java +++ /dev/null @@ -1,26 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.seatunnel.connectors.seatunnel.rabbitmq.source; - - -import java.io.Serializable; - -public class RabbitMQSplitEnumeratorState implements Serializable { - -} diff --git a/seatunnel-connectors-v2/connector-rabbitmq/src/main/java/org/apache/seatunnel/connectors/seatunnel/rabbitmq/source/RabbitMQSplitReaderThread.java b/seatunnel-connectors-v2/connector-rabbitmq/src/main/java/org/apache/seatunnel/connectors/seatunnel/rabbitmq/source/RabbitMQSplitReaderThread.java deleted file mode 100644 index 1226afe1460..00000000000 --- a/seatunnel-connectors-v2/connector-rabbitmq/src/main/java/org/apache/seatunnel/connectors/seatunnel/rabbitmq/source/RabbitMQSplitReaderThread.java +++ /dev/null @@ -1,35 +0,0 @@ -package org.apache.seatunnel.connectors.seatunnel.rabbitmq.source; - -import com.rabbitmq.client.Consumer; -import com.rabbitmq.client.Delivery; -import org.apache.seatunnel.common.Handover; -import org.apache.seatunnel.connectors.seatunnel.pulsar.source.reader.PulsarSplitReaderThread; -import org.apache.seatunnel.connectors.seatunnel.pulsar.source.reader.RecordWithSplitId; -import org.apache.seatunnel.connectors.seatunnel.rabbitmq.client.QueueingConsumer; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -public class RabbitMQSplitReaderThread extends Thread implements Cloneable { - private static final Logger LOG = LoggerFactory.getLogger(RabbitMQSplitReaderThread.class); - protected Consumer consumer; - protected final Handover handover; - private volatile boolean running; - - public RabbitMQSplitReaderThread(Handover handover) { - this.handover = handover; - } - - - public void open(Handover handover) { - consumer = new QueueingConsumer(null); - - } - - @Override - public void run() { - while (running) { - - } - - } -} diff --git a/seatunnel-connectors-v2/connector-rabbitmq/src/main/java/org/apache/seatunnel/connectors/seatunnel/rabbitmq/source/RabbitMQSource.java b/seatunnel-connectors-v2/connector-rabbitmq/src/main/java/org/apache/seatunnel/connectors/seatunnel/rabbitmq/source/RabbitmqSource.java similarity index 58% rename from seatunnel-connectors-v2/connector-rabbitmq/src/main/java/org/apache/seatunnel/connectors/seatunnel/rabbitmq/source/RabbitMQSource.java rename to seatunnel-connectors-v2/connector-rabbitmq/src/main/java/org/apache/seatunnel/connectors/seatunnel/rabbitmq/source/RabbitmqSource.java index efbcd7c39f7..5d430219759 100644 --- a/seatunnel-connectors-v2/connector-rabbitmq/src/main/java/org/apache/seatunnel/connectors/seatunnel/rabbitmq/source/RabbitMQSource.java +++ b/seatunnel-connectors-v2/connector-rabbitmq/src/main/java/org/apache/seatunnel/connectors/seatunnel/rabbitmq/source/RabbitmqSource.java @@ -17,7 +17,6 @@ package org.apache.seatunnel.connectors.seatunnel.rabbitmq.source; -import com.google.auto.service.AutoService; import org.apache.seatunnel.api.common.JobContext; import org.apache.seatunnel.api.common.PrepareFailException; import org.apache.seatunnel.api.serialization.DeserializationSchema; @@ -25,23 +24,29 @@ import org.apache.seatunnel.api.source.SeaTunnelSource; import org.apache.seatunnel.api.source.SourceReader; import org.apache.seatunnel.api.source.SourceSplitEnumerator; +import org.apache.seatunnel.api.table.type.SeaTunnelDataType; import org.apache.seatunnel.api.table.type.SeaTunnelRow; import org.apache.seatunnel.api.table.type.SeaTunnelRowType; -import org.apache.seatunnel.common.constants.JobMode; -import org.apache.seatunnel.connectors.seatunnel.rabbitmq.split.RabbitMQSplit; -import org.apache.seatunnel.connectors.seatunnel.rabbitmq.split.RabbitMQState; +import org.apache.seatunnel.connectors.seatunnel.common.schema.SeaTunnelSchema; +import org.apache.seatunnel.connectors.seatunnel.rabbitmq.config.RabbitmqConfig; +import org.apache.seatunnel.connectors.seatunnel.rabbitmq.split.RabbitmqSplit; +import org.apache.seatunnel.connectors.seatunnel.rabbitmq.split.RabbitmqSplitEnumeratorState; +import org.apache.seatunnel.format.json.JsonDeserializationSchema; + import org.apache.seatunnel.shade.com.typesafe.config.Config; +import com.google.auto.service.AutoService; + @AutoService(SeaTunnelSource.class) -public class RabbitMQSource implements SeaTunnelSource { +public class RabbitmqSource implements SeaTunnelSource { private DeserializationSchema deserializationSchema; - private SeaTunnelRowType typeInfo; private JobContext jobContext; + private RabbitmqConfig rabbitMQConfig; @Override public Boundedness getBoundedness() { - return JobMode.BATCH.equals(jobContext.getJobMode()) ? Boundedness.BOUNDED : Boundedness.UNBOUNDED; + return Boundedness.BOUNDED; } @Override @@ -51,36 +56,39 @@ public String getPluginName() { @Override public void prepare(Config config) throws PrepareFailException { - + this.rabbitMQConfig = new RabbitmqConfig(config); + setDeserialization(config); } @Override - public SeaTunnelRowType getProducedType() { - return this.typeInfo; + public SeaTunnelDataType getProducedType() { + return deserializationSchema.getProducedType(); } @Override - public SourceReader createReader(SourceReader.Context readerContext) throws Exception { - return null; + public SourceReader createReader(SourceReader.Context readerContext) throws Exception { + return new RabbitmqSourceReader(deserializationSchema, readerContext, rabbitMQConfig); } @Override - public SourceSplitEnumerator createEnumerator(SourceSplitEnumerator.Context enumeratorContext) throws Exception { - return null; + public SourceSplitEnumerator createEnumerator(SourceSplitEnumerator.Context enumeratorContext) throws Exception { + return new RabbitmqSplitEnumerator(); } @Override - public SourceSplitEnumerator restoreEnumerator(SourceSplitEnumerator.Context enumeratorContext, RabbitMQState checkpointState) throws Exception { - return null; + public SourceSplitEnumerator restoreEnumerator(SourceSplitEnumerator.Context enumeratorContext, RabbitmqSplitEnumeratorState checkpointState) throws Exception { + return new RabbitmqSplitEnumerator(); } - @Override public void setJobContext(JobContext jobContext) { this.jobContext = jobContext; } private void setDeserialization(Config config) { - + // TODO: format SPI + //only support json deserializationSchema + SeaTunnelRowType rowType = SeaTunnelSchema.buildWithConfig(config.getConfig(SeaTunnelSchema.SCHEMA)).getSeaTunnelRowType(); + this.deserializationSchema = new JsonDeserializationSchema(false, false, rowType); } } diff --git a/seatunnel-connectors-v2/connector-rabbitmq/src/main/java/org/apache/seatunnel/connectors/seatunnel/rabbitmq/source/RabbitmqSourceReader.java b/seatunnel-connectors-v2/connector-rabbitmq/src/main/java/org/apache/seatunnel/connectors/seatunnel/rabbitmq/source/RabbitmqSourceReader.java new file mode 100644 index 00000000000..2f3021df26d --- /dev/null +++ b/seatunnel-connectors-v2/connector-rabbitmq/src/main/java/org/apache/seatunnel/connectors/seatunnel/rabbitmq/source/RabbitmqSourceReader.java @@ -0,0 +1,205 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.connectors.seatunnel.rabbitmq.source; + +import org.apache.seatunnel.api.serialization.DeserializationSchema; +import org.apache.seatunnel.api.source.Boundedness; +import org.apache.seatunnel.api.source.Collector; +import org.apache.seatunnel.api.source.SourceReader; +import org.apache.seatunnel.api.table.type.SeaTunnelRow; +import org.apache.seatunnel.common.Handover; +import org.apache.seatunnel.connectors.seatunnel.rabbitmq.client.RabbitmqClient; +import org.apache.seatunnel.connectors.seatunnel.rabbitmq.config.RabbitmqConfig; +import org.apache.seatunnel.connectors.seatunnel.rabbitmq.split.RabbitmqSplit; + +import com.rabbitmq.client.AMQP; +import com.rabbitmq.client.Channel; +import com.rabbitmq.client.DefaultConsumer; +import com.rabbitmq.client.Delivery; +import com.rabbitmq.client.Envelope; +import lombok.extern.slf4j.Slf4j; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashSet; +import java.util.List; +import java.util.Optional; +import java.util.Set; +import java.util.SortedMap; +import java.util.TreeMap; + +@Slf4j +public class RabbitmqSourceReader implements SourceReader { + protected final Handover handover; + + protected final SourceReader.Context context; + protected transient Channel channel; + private final boolean usesCorrelationId = true; + protected transient boolean autoAck; + + protected transient Set correlationIdsProcessedButNotAcknowledged; + protected transient List deliveryTagsProcessedForCurrentSnapshot; + + protected final SortedMap> pendingDeliveryTagsToCommit; + protected final SortedMap> pendingCorrelationIdsToCommit; + + private final DeserializationSchema deserializationSchema; + private RabbitmqClient rabbitMQClient; + private DefaultConsumer consumer; + private final RabbitmqConfig config; + + public RabbitmqSourceReader(DeserializationSchema deserializationSchema, + SourceReader.Context context, + RabbitmqConfig config) { + this.handover = new Handover<>(); + this.pendingDeliveryTagsToCommit = Collections.synchronizedSortedMap(new TreeMap<>()); + this.pendingCorrelationIdsToCommit = Collections.synchronizedSortedMap(new TreeMap<>()); + this.context = context; + this.deserializationSchema = deserializationSchema; + this.config = config; + this.rabbitMQClient = new RabbitmqClient(config); + this.channel = rabbitMQClient.getChannel(); + } + + @Override + public void open() throws Exception { + this.correlationIdsProcessedButNotAcknowledged = new HashSet<>(); + this.deliveryTagsProcessedForCurrentSnapshot = new ArrayList<>(); + consumer = rabbitMQClient.getQueueingConsumer(handover); + + if (Boundedness.UNBOUNDED.equals(context.getBoundedness())) { + autoAck = false; + // enables transaction mode + channel.txSelect(); + } else { + autoAck = true; + } + + log.debug("Starting RabbitMQ source with autoAck status: " + autoAck); + channel.basicConsume(config.getQueueName(), autoAck, consumer); + } + + @Override + public void close() throws IOException { + + } + + @Override + public void pollNext(Collector output) throws Exception { + Optional deliveryOptional = handover.pollNext(); + if (deliveryOptional.isPresent()) { + Delivery delivery = deliveryOptional.get(); + AMQP.BasicProperties properties = delivery.getProperties(); + byte[] body = delivery.getBody(); + Envelope envelope = delivery.getEnvelope(); + synchronized (output.getCheckpointLock()) { + boolean newMessage = verifyMessageIdentifier(properties.getCorrelationId(), envelope.getDeliveryTag()); + if (!newMessage) { + return; + } + deliveryTagsProcessedForCurrentSnapshot.add(envelope.getDeliveryTag()); + deserializationSchema.deserialize(body, output); + } + } + } + + @Override + public List snapshotState(long checkpointId) throws Exception { + + List pendingSplit = Collections.singletonList(new RabbitmqSplit(deliveryTagsProcessedForCurrentSnapshot, correlationIdsProcessedButNotAcknowledged)); + // perform a snapshot for these splits. + List deliveryTags = + pendingDeliveryTagsToCommit.computeIfAbsent(checkpointId, id -> new ArrayList<>()); + Set correlationIds = + pendingCorrelationIdsToCommit.computeIfAbsent(checkpointId, id -> new HashSet<>()); + // put currentCheckPoint deliveryTags and CorrelationIds. + for (RabbitmqSplit split : pendingSplit) { + List currentCheckPointDeliveryTags = split.getDeliveryTags(); + Set currentCheckPointCorrelationIds = split.getCorrelationIds(); + + if (currentCheckPointDeliveryTags != null) { + deliveryTags.addAll(currentCheckPointDeliveryTags); + } + if (currentCheckPointCorrelationIds != null) { + correlationIds.addAll(currentCheckPointCorrelationIds); + } + } + return pendingSplit; + } + + @Override + public void addSplits(List splits) { + + } + + @Override + public void handleNoMoreSplits() { + } + + @Override + public void notifyCheckpointComplete(long checkpointId) throws Exception { + log.debug("Committing cursors for checkpoint {}", checkpointId); + List pendingDeliveryTags = pendingDeliveryTagsToCommit.remove(checkpointId); + Set pendingCorrelationIds = pendingCorrelationIdsToCommit.remove(checkpointId); + + if (pendingDeliveryTags == null || pendingCorrelationIds == null) { + log.debug( + "pending delivery tags or correlationIds checkpoint {} either do not exist or have already been committed.", + checkpointId); + return; + } + acknowledgeDeliveryTags(pendingDeliveryTags); + correlationIdsProcessedButNotAcknowledged.removeAll(pendingCorrelationIds); + + } + + protected void acknowledgeDeliveryTags(List deliveryTags) { + try { + for (long id : deliveryTags) { + channel.basicAck(id, false); + } + channel.txCommit(); + } catch (IOException e) { + throw new RuntimeException( + "Messages could not be acknowledged during checkpoint creation.", e); + } + } + + public boolean verifyMessageIdentifier(String correlationId, long deliveryTag) { + if (!autoAck) { + if (usesCorrelationId) { + com.google.common.base.Preconditions.checkNotNull( + correlationId, + "RabbitMQ source was instantiated with usesCorrelationId set to " + + "true yet we couldn't extract the correlation id from it!"); + if (!correlationIdsProcessedButNotAcknowledged.add(correlationId)) { + // we have already processed this message + try { + channel.basicReject(deliveryTag, false); + } catch (IOException e) { + throw new RuntimeException( + "Message could not be acknowledged with basicReject.", e); + } + return false; + } + } + } + return true; + } +} diff --git a/seatunnel-connectors-v2/connector-rabbitmq/src/main/java/org/apache/seatunnel/connectors/seatunnel/rabbitmq/source/RabbitMQSplitEnumerator.java b/seatunnel-connectors-v2/connector-rabbitmq/src/main/java/org/apache/seatunnel/connectors/seatunnel/rabbitmq/source/RabbitmqSplitEnumerator.java similarity index 95% rename from seatunnel-connectors-v2/connector-rabbitmq/src/main/java/org/apache/seatunnel/connectors/seatunnel/rabbitmq/source/RabbitMQSplitEnumerator.java rename to seatunnel-connectors-v2/connector-rabbitmq/src/main/java/org/apache/seatunnel/connectors/seatunnel/rabbitmq/source/RabbitmqSplitEnumerator.java index 89d88b0d18c..b912f4e4109 100644 --- a/seatunnel-connectors-v2/connector-rabbitmq/src/main/java/org/apache/seatunnel/connectors/seatunnel/rabbitmq/source/RabbitMQSplitEnumerator.java +++ b/seatunnel-connectors-v2/connector-rabbitmq/src/main/java/org/apache/seatunnel/connectors/seatunnel/rabbitmq/source/RabbitmqSplitEnumerator.java @@ -22,8 +22,7 @@ import java.io.IOException; import java.util.List; - -public class RabbitMQSplitEnumerator implements SourceSplitEnumerator { +public class RabbitmqSplitEnumerator implements SourceSplitEnumerator { @Override public void open() { @@ -67,6 +66,6 @@ public Object snapshotState(long checkpointId) throws Exception { @Override public void notifyCheckpointComplete(long checkpointId) throws Exception { - + // nothing } } diff --git a/seatunnel-connectors-v2/connector-rabbitmq/src/main/java/org/apache/seatunnel/connectors/seatunnel/rabbitmq/split/RabbitMQState.java b/seatunnel-connectors-v2/connector-rabbitmq/src/main/java/org/apache/seatunnel/connectors/seatunnel/rabbitmq/split/RabbitMQState.java deleted file mode 100644 index b545856c406..00000000000 --- a/seatunnel-connectors-v2/connector-rabbitmq/src/main/java/org/apache/seatunnel/connectors/seatunnel/rabbitmq/split/RabbitMQState.java +++ /dev/null @@ -1,6 +0,0 @@ -package org.apache.seatunnel.connectors.seatunnel.rabbitmq.split; - -import java.io.Serializable; - -public class RabbitMQState implements Serializable { -} diff --git a/seatunnel-connectors-v2/connector-rabbitmq/src/main/java/org/apache/seatunnel/connectors/seatunnel/rabbitmq/split/RabbitMQSplit.java b/seatunnel-connectors-v2/connector-rabbitmq/src/main/java/org/apache/seatunnel/connectors/seatunnel/rabbitmq/split/RabbitmqSplit.java similarity index 85% rename from seatunnel-connectors-v2/connector-rabbitmq/src/main/java/org/apache/seatunnel/connectors/seatunnel/rabbitmq/split/RabbitMQSplit.java rename to seatunnel-connectors-v2/connector-rabbitmq/src/main/java/org/apache/seatunnel/connectors/seatunnel/rabbitmq/split/RabbitmqSplit.java index 6de8eeaa04c..70031aa6673 100644 --- a/seatunnel-connectors-v2/connector-rabbitmq/src/main/java/org/apache/seatunnel/connectors/seatunnel/rabbitmq/split/RabbitMQSplit.java +++ b/seatunnel-connectors-v2/connector-rabbitmq/src/main/java/org/apache/seatunnel/connectors/seatunnel/rabbitmq/split/RabbitmqSplit.java @@ -17,28 +17,24 @@ package org.apache.seatunnel.connectors.seatunnel.rabbitmq.split; -import com.sun.istack.internal.Nullable; +import org.apache.seatunnel.api.source.SourceSplit; + import lombok.AllArgsConstructor; import lombok.Getter; import lombok.Setter; -import org.apache.seatunnel.api.source.SourceSplit; - import java.util.List; +import java.util.Set; @Getter @Setter @AllArgsConstructor -public class RabbitMQSplit implements SourceSplit { - @Nullable +public class RabbitmqSplit implements SourceSplit { private List deliveryTags; + private Set correlationIds; @Override public String splitId() { return ""; } - public RabbitMQSplit copy() { - return new RabbitMQSplit(deliveryTags); - } - } diff --git a/seatunnel-connectors-v2/connector-rabbitmq/src/main/java/org/apache/seatunnel/connectors/seatunnel/rabbitmq/split/RabbitmqSplitEnumeratorState.java b/seatunnel-connectors-v2/connector-rabbitmq/src/main/java/org/apache/seatunnel/connectors/seatunnel/rabbitmq/split/RabbitmqSplitEnumeratorState.java new file mode 100644 index 00000000000..d4aedf1248c --- /dev/null +++ b/seatunnel-connectors-v2/connector-rabbitmq/src/main/java/org/apache/seatunnel/connectors/seatunnel/rabbitmq/split/RabbitmqSplitEnumeratorState.java @@ -0,0 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.connectors.seatunnel.rabbitmq.split; + +import java.io.Serializable; + +public class RabbitmqSplitEnumeratorState implements Serializable { +} diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-rabbitmq-e2e/pom.xml b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-rabbitmq-e2e/pom.xml new file mode 100644 index 00000000000..fb3d0cb3c58 --- /dev/null +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-rabbitmq-e2e/pom.xml @@ -0,0 +1,43 @@ + + + + + org.apache.seatunnel + seatunnel-connector-v2-e2e + ${revision} + + 4.0.0 + + connector-rabbitmq-e2e + + + + + org.apache.seatunnel + connector-rabbitmq + ${project.version} + test + + + org.apache.seatunnel + connector-console + ${project.version} + test + + + \ No newline at end of file diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-rabbitmq-e2e/src/test/java/org/apache/seatunnel/e2e/connector/rabbitmq/RabbitmqIT.java b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-rabbitmq-e2e/src/test/java/org/apache/seatunnel/e2e/connector/rabbitmq/RabbitmqIT.java new file mode 100644 index 00000000000..764d6f53208 --- /dev/null +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-rabbitmq-e2e/src/test/java/org/apache/seatunnel/e2e/connector/rabbitmq/RabbitmqIT.java @@ -0,0 +1,203 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.e2e.connector.rabbitmq; + +import org.apache.seatunnel.api.table.type.ArrayType; +import org.apache.seatunnel.api.table.type.BasicType; +import org.apache.seatunnel.api.table.type.DecimalType; +import org.apache.seatunnel.api.table.type.LocalTimeType; +import org.apache.seatunnel.api.table.type.MapType; +import org.apache.seatunnel.api.table.type.PrimitiveByteArrayType; +import org.apache.seatunnel.api.table.type.SeaTunnelDataType; +import org.apache.seatunnel.api.table.type.SeaTunnelRow; +import org.apache.seatunnel.api.table.type.SeaTunnelRowType; +import org.apache.seatunnel.e2e.common.TestResource; +import org.apache.seatunnel.e2e.common.TestSuiteBase; +import org.apache.seatunnel.e2e.common.container.TestContainer; +import org.apache.seatunnel.format.json.JsonSerializationSchema; + +import com.rabbitmq.client.Channel; +import com.rabbitmq.client.Connection; +import com.rabbitmq.client.ConnectionFactory; +import lombok.extern.slf4j.Slf4j; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.TestTemplate; +import org.testcontainers.containers.Container; +import org.testcontainers.containers.GenericContainer; +import org.testcontainers.containers.output.Slf4jLogConsumer; +import org.testcontainers.containers.wait.strategy.HostPortWaitStrategy; +import org.testcontainers.lifecycle.Startables; +import org.testcontainers.utility.DockerImageName; +import org.testcontainers.utility.DockerLoggerFactory; + +import java.io.IOException; +import java.math.BigDecimal; +import java.nio.charset.StandardCharsets; +import java.time.Duration; +import java.time.LocalDate; +import java.time.LocalDateTime; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.stream.Stream; + +import scala.Tuple2; + +@Slf4j +public class RabbitmqIT extends TestSuiteBase implements TestResource { + private static final String IMAGE = "library/rabbitmq:3.11"; + private static final String HOST = "rabbitmq-e2e"; + private static final int PORT = 5672; + private static final String QUEUE_NAME = "test"; + private static final String USERNAME = "guest"; + private static final String PASSWORD = "guest"; + + private static final Tuple2> TEST_DATASET = generateTestDataSet(); + + private GenericContainer rabbitmqContainer; + Connection connection = null; + Channel channel = null; + + @BeforeAll + @Override + public void startUp() throws Exception { + this.rabbitmqContainer = new GenericContainer<>(DockerImageName.parse(IMAGE)) + .withNetwork(NETWORK) + .withNetworkAliases(HOST) + .withExposedPorts(PORT) + .withLogConsumer(new Slf4jLogConsumer(DockerLoggerFactory.getLogger(IMAGE))) + .waitingFor(new HostPortWaitStrategy() + .withStartupTimeout(Duration.ofMinutes(2))); + Startables.deepStart(Stream.of(rabbitmqContainer)).join(); + log.info("rabbitmq container started"); + this.initRabbitMQ(); + this.initSourceData(); + } + + private void initSourceData() throws IOException { + JsonSerializationSchema jsonSerializationSchema = new JsonSerializationSchema(TEST_DATASET._1()); + List rows = TEST_DATASET._2(); + for (int i = 0; i < rows.size(); i++) { + channel.basicPublish("", "", null, new String(jsonSerializationSchema.serialize(rows.get(i))).getBytes(StandardCharsets.UTF_8)); + } + } + + private static Tuple2> generateTestDataSet() { + + SeaTunnelRowType rowType = new SeaTunnelRowType( + new String[]{ + "id", + "c_map", + "c_array", + "c_string", + "c_boolean", + "c_tinyint", + "c_smallint", + "c_int", + "c_bigint", + "c_float", + "c_double", + "c_decimal", + "c_bytes", + "c_date", + "c_timestamp" + }, + new SeaTunnelDataType[]{ + BasicType.LONG_TYPE, + new MapType(BasicType.STRING_TYPE, BasicType.SHORT_TYPE), + ArrayType.BYTE_ARRAY_TYPE, + BasicType.STRING_TYPE, + BasicType.BOOLEAN_TYPE, + BasicType.BYTE_TYPE, + BasicType.SHORT_TYPE, + BasicType.INT_TYPE, + BasicType.LONG_TYPE, + BasicType.FLOAT_TYPE, + BasicType.DOUBLE_TYPE, + new DecimalType(2, 1), + PrimitiveByteArrayType.INSTANCE, + LocalTimeType.LOCAL_DATE_TYPE, + LocalTimeType.LOCAL_DATE_TIME_TYPE + } + ); + + List rows = new ArrayList<>(); + for (int i = 0; i < 100; i++) { + SeaTunnelRow row = new SeaTunnelRow( + new Object[]{ + Long.valueOf(i), + Collections.singletonMap("key", Short.parseShort("1")), + new Byte[]{Byte.parseByte("1")}, + "string", + Boolean.FALSE, + Byte.parseByte("1"), + Short.parseShort("1"), + Integer.parseInt("1"), + Long.parseLong("1"), + Float.parseFloat("1.1"), + Double.parseDouble("1.1"), + BigDecimal.valueOf(11, 1), + "test".getBytes(), + LocalDate.now(), + LocalDateTime.now() + }); + rows.add(row); + } + return Tuple2.apply(rowType, rows); + } + + private void initRabbitMQ() { + + try { + System.out.println(rabbitmqContainer.getHost()); + System.out.println(rabbitmqContainer.getFirstMappedPort()); + + ConnectionFactory connectionFactory = new ConnectionFactory(); + connectionFactory.setHost(rabbitmqContainer.getHost()); + connectionFactory.setPort(rabbitmqContainer.getFirstMappedPort()); + connectionFactory.setUsername(USERNAME); + connectionFactory.setPassword(PASSWORD); + connection = connectionFactory.newConnection(); + channel = connection.createChannel(); + channel.queueDeclare(QUEUE_NAME, true, false, false, null); + } catch (Exception e) { + throw new RuntimeException("init Rabbitmq error", e); + } + } + + @AfterAll + @Override + public void tearDown() throws Exception { + if (channel != null) { + channel.close(); + } + if (connection != null) { + connection.close(); + } + rabbitmqContainer.close(); + } + + @TestTemplate + public void testRabbitMQ(TestContainer container) throws IOException, InterruptedException { + Container.ExecResult execResult = container.executeJob("/rabbitmq-to-rabbitmq.conf"); + Assertions.assertEquals(0, execResult.getExitCode()); + + } +} diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-rabbitmq-e2e/src/test/resources/log4j.properties b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-rabbitmq-e2e/src/test/resources/log4j.properties new file mode 100644 index 00000000000..db5d9e51220 --- /dev/null +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-rabbitmq-e2e/src/test/resources/log4j.properties @@ -0,0 +1,22 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Set everything to be logged to the console +log4j.rootCategory=INFO, console +log4j.appender.console=org.apache.log4j.ConsoleAppender +log4j.appender.console.target=System.err +log4j.appender.console.layout=org.apache.log4j.PatternLayout +log4j.appender.console.layout.ConversionPattern=%d{yy/MM/dd HH:mm:ss} %p %c{1}: %m%n diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-rabbitmq-e2e/src/test/resources/rabbitmq-to-rabbitmq.conf b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-rabbitmq-e2e/src/test/resources/rabbitmq-to-rabbitmq.conf new file mode 100644 index 00000000000..c91471ba8d5 --- /dev/null +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-rabbitmq-e2e/src/test/resources/rabbitmq-to-rabbitmq.conf @@ -0,0 +1,59 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +env { + execution.parallelism = 1 + job.mode = "BATCH" +} + +source { + RabbitMQ { + host = "rabbitmq-e2e" + port = 5672 + virtual_host = "/" + username = "guest" + password = "guest" + queue_name = "test" + schema = { + fields { + id = bigint + c_map = "map" + c_array = "array" + c_string = string + c_boolean = boolean + c_tinyint = tinyint + c_smallint = smallint + c_int = int + c_bigint = bigint + c_float = float + c_double = double + c_decimal = "decimal(2, 1)" + c_bytes = bytes + c_date = date + c_timestamp = timestamp + } + } + format = "json" + } +} + +transform { +} + +sink { + Console {} +} \ No newline at end of file diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/pom.xml b/seatunnel-e2e/seatunnel-connector-v2-e2e/pom.xml index 1fbf57b90bf..8c4bcb8a09b 100644 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/pom.xml +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/pom.xml @@ -28,6 +28,7 @@ connector-jdbc-e2e connector-redis-e2e connector-clickhouse-e2e + connector-rabbitmq-e2e seatunnel-connector-v2-e2e From 26221585e2da7f3d82ea24e436329a736ba9ab3b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=AF=95=E5=8D=9A?= Date: Tue, 1 Nov 2022 23:05:15 +0800 Subject: [PATCH 04/15] [Feature][Connector-V2] add rabbitmq connector --- .../rabbitmq/config/RabbitmqConfig.java | 5 ++ .../e2e/connector/rabbitmq/RabbitmqIT.java | 82 ++++++++++++++----- .../test/resources/rabbitmq-to-rabbitmq.conf | 9 +- 3 files changed, 74 insertions(+), 22 deletions(-) diff --git a/seatunnel-connectors-v2/connector-rabbitmq/src/main/java/org/apache/seatunnel/connectors/seatunnel/rabbitmq/config/RabbitmqConfig.java b/seatunnel-connectors-v2/connector-rabbitmq/src/main/java/org/apache/seatunnel/connectors/seatunnel/rabbitmq/config/RabbitmqConfig.java index 3192d52612d..fd36c01aba0 100644 --- a/seatunnel-connectors-v2/connector-rabbitmq/src/main/java/org/apache/seatunnel/connectors/seatunnel/rabbitmq/config/RabbitmqConfig.java +++ b/seatunnel-connectors-v2/connector-rabbitmq/src/main/java/org/apache/seatunnel/connectors/seatunnel/rabbitmq/config/RabbitmqConfig.java @@ -4,6 +4,7 @@ import org.apache.seatunnel.shade.com.typesafe.config.Config; +import com.google.common.annotations.VisibleForTesting; import lombok.AllArgsConstructor; import lombok.Getter; import lombok.Setter; @@ -115,6 +116,10 @@ public RabbitmqConfig(Config config) { this.exchange = config.getString(EXCHANGE); } parseSinkOptionProperties(config); + } + + @VisibleForTesting + public RabbitmqConfig() { } } diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-rabbitmq-e2e/src/test/java/org/apache/seatunnel/e2e/connector/rabbitmq/RabbitmqIT.java b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-rabbitmq-e2e/src/test/java/org/apache/seatunnel/e2e/connector/rabbitmq/RabbitmqIT.java index 764d6f53208..de267c4a14a 100644 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-rabbitmq-e2e/src/test/java/org/apache/seatunnel/e2e/connector/rabbitmq/RabbitmqIT.java +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-rabbitmq-e2e/src/test/java/org/apache/seatunnel/e2e/connector/rabbitmq/RabbitmqIT.java @@ -17,6 +17,7 @@ package org.apache.seatunnel.e2e.connector.rabbitmq; +import com.rabbitmq.client.*; import org.apache.seatunnel.api.table.type.ArrayType; import org.apache.seatunnel.api.table.type.BasicType; import org.apache.seatunnel.api.table.type.DecimalType; @@ -26,14 +27,14 @@ import org.apache.seatunnel.api.table.type.SeaTunnelDataType; import org.apache.seatunnel.api.table.type.SeaTunnelRow; import org.apache.seatunnel.api.table.type.SeaTunnelRowType; +import org.apache.seatunnel.common.Handover; +import org.apache.seatunnel.connectors.seatunnel.rabbitmq.client.RabbitmqClient; +import org.apache.seatunnel.connectors.seatunnel.rabbitmq.config.RabbitmqConfig; import org.apache.seatunnel.e2e.common.TestResource; import org.apache.seatunnel.e2e.common.TestSuiteBase; import org.apache.seatunnel.e2e.common.container.TestContainer; import org.apache.seatunnel.format.json.JsonSerializationSchema; -import com.rabbitmq.client.Channel; -import com.rabbitmq.client.Connection; -import com.rabbitmq.client.ConnectionFactory; import lombok.extern.slf4j.Slf4j; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.Assertions; @@ -56,24 +57,30 @@ import java.util.ArrayList; import java.util.Collections; import java.util.List; +import java.util.Optional; import java.util.stream.Stream; import scala.Tuple2; @Slf4j public class RabbitmqIT extends TestSuiteBase implements TestResource { - private static final String IMAGE = "library/rabbitmq:3.11"; + private static final String IMAGE = "rabbitmq:3-management"; private static final String HOST = "rabbitmq-e2e"; private static final int PORT = 5672; private static final String QUEUE_NAME = "test"; private static final String USERNAME = "guest"; private static final String PASSWORD = "guest"; + Handover handover = new Handover<>(); + private static final Tuple2> TEST_DATASET = generateTestDataSet(); private GenericContainer rabbitmqContainer; Connection connection = null; Channel channel = null; + Channel sinkChannel = null; + RabbitmqClient rabbitmqClient; + RabbitmqClient sinkRabbitmqClient; @BeforeAll @Override @@ -81,7 +88,7 @@ public void startUp() throws Exception { this.rabbitmqContainer = new GenericContainer<>(DockerImageName.parse(IMAGE)) .withNetwork(NETWORK) .withNetworkAliases(HOST) - .withExposedPorts(PORT) + .withExposedPorts(PORT, 15672) .withLogConsumer(new Slf4jLogConsumer(DockerLoggerFactory.getLogger(IMAGE))) .waitingFor(new HostPortWaitStrategy() .withStartupTimeout(Duration.ofMinutes(2))); @@ -89,14 +96,15 @@ public void startUp() throws Exception { log.info("rabbitmq container started"); this.initRabbitMQ(); this.initSourceData(); + this.initSinkRabbitMQ(); } - private void initSourceData() throws IOException { + private void initSourceData() throws IOException, InterruptedException { JsonSerializationSchema jsonSerializationSchema = new JsonSerializationSchema(TEST_DATASET._1()); List rows = TEST_DATASET._2(); - for (int i = 0; i < rows.size(); i++) { - channel.basicPublish("", "", null, new String(jsonSerializationSchema.serialize(rows.get(i))).getBytes(StandardCharsets.UTF_8)); - } + for (int i = 0; i < rows.size(); i++) { + rabbitmqClient.write(new String(jsonSerializationSchema.serialize(rows.get(i))).getBytes(StandardCharsets.UTF_8)); + } } private static Tuple2> generateTestDataSet() { @@ -166,17 +174,31 @@ private static Tuple2> generateTestDataSet( private void initRabbitMQ() { try { - System.out.println(rabbitmqContainer.getHost()); - System.out.println(rabbitmqContainer.getFirstMappedPort()); - - ConnectionFactory connectionFactory = new ConnectionFactory(); - connectionFactory.setHost(rabbitmqContainer.getHost()); - connectionFactory.setPort(rabbitmqContainer.getFirstMappedPort()); - connectionFactory.setUsername(USERNAME); - connectionFactory.setPassword(PASSWORD); - connection = connectionFactory.newConnection(); - channel = connection.createChannel(); - channel.queueDeclare(QUEUE_NAME, true, false, false, null); + RabbitmqConfig config = new RabbitmqConfig(); + config.setHost(rabbitmqContainer.getHost()); + config.setPort(rabbitmqContainer.getFirstMappedPort()); + config.setQueueName("test"); + config.setVirtualHost("/"); + config.setUsername("guest"); + config.setPassword("guest"); + rabbitmqClient = new RabbitmqClient(config); + } catch (Exception e) { + throw new RuntimeException("init Rabbitmq error", e); + } + } + + private void initSinkRabbitMQ() { + + try { + RabbitmqConfig config = new RabbitmqConfig(); + config.setHost(rabbitmqContainer.getHost()); + config.setPort(rabbitmqContainer.getFirstMappedPort()); + config.setQueueName("test1"); + config.setVirtualHost("/"); + config.setUsername("guest"); + config.setPassword("guest"); + sinkRabbitmqClient = new RabbitmqClient(config); + sinkChannel = sinkRabbitmqClient.getChannel(); } catch (Exception e) { throw new RuntimeException("init Rabbitmq error", e); } @@ -195,9 +217,27 @@ public void tearDown() throws Exception { } @TestTemplate - public void testRabbitMQ(TestContainer container) throws IOException, InterruptedException { + public void testRabbitMQ(TestContainer container) throws Exception { Container.ExecResult execResult = container.executeJob("/rabbitmq-to-rabbitmq.conf"); Assertions.assertEquals(0, execResult.getExitCode()); + RabbitmqConfig config = new RabbitmqConfig(); + config.setHost(rabbitmqContainer.getHost()); + config.setPort(rabbitmqContainer.getFirstMappedPort()); + config.setQueueName("test1"); + config.setVirtualHost("/"); + config.setUsername("guest"); + config.setPassword("guest"); + + DefaultConsumer consumer = sinkRabbitmqClient.getQueueingConsumer(handover); + sinkChannel.basicConsume(config.getQueueName(), true, consumer); + for(int i = 0; i < 100; i++) { + Optional deliveryOptional = handover.pollNext(); + if (deliveryOptional.isPresent()) { + Delivery delivery = deliveryOptional.get(); + byte[] body = delivery.getBody(); + System.out.println(new String(body)); + } + } } } diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-rabbitmq-e2e/src/test/resources/rabbitmq-to-rabbitmq.conf b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-rabbitmq-e2e/src/test/resources/rabbitmq-to-rabbitmq.conf index c91471ba8d5..0d5818f6465 100644 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-rabbitmq-e2e/src/test/resources/rabbitmq-to-rabbitmq.conf +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-rabbitmq-e2e/src/test/resources/rabbitmq-to-rabbitmq.conf @@ -17,7 +17,6 @@ env { execution.parallelism = 1 - job.mode = "BATCH" } source { @@ -56,4 +55,12 @@ transform { sink { Console {} + RabbitMQ { + host = "rabbitmq-e2e" + port = 5672 + virtual_host = "/" + username = "guest" + password = "guest" + queue_name = "test1" + } } \ No newline at end of file From 348751e139c45c65f21b113cc189799e9d749301 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=AF=95=E5=8D=9A?= Date: Fri, 4 Nov 2022 16:10:52 +0800 Subject: [PATCH 05/15] [Feature][Connector-V2] add rabbitmq connector --- .../rabbitmq/client/QueueingConsumer.java | 2 +- .../rabbitmq/source/RabbitmqSource.java | 3 +- .../rabbitmq/source/RabbitmqSourceReader.java | 8 +- .../e2e/connector/rabbitmq/RabbitmqIT.java | 75 +++++++++++-------- .../test/resources/rabbitmq-to-rabbitmq.conf | 2 +- 5 files changed, 56 insertions(+), 34 deletions(-) diff --git a/seatunnel-connectors-v2/connector-rabbitmq/src/main/java/org/apache/seatunnel/connectors/seatunnel/rabbitmq/client/QueueingConsumer.java b/seatunnel-connectors-v2/connector-rabbitmq/src/main/java/org/apache/seatunnel/connectors/seatunnel/rabbitmq/client/QueueingConsumer.java index c77e81fce26..36a27cce5be 100644 --- a/seatunnel-connectors-v2/connector-rabbitmq/src/main/java/org/apache/seatunnel/connectors/seatunnel/rabbitmq/client/QueueingConsumer.java +++ b/seatunnel-connectors-v2/connector-rabbitmq/src/main/java/org/apache/seatunnel/connectors/seatunnel/rabbitmq/client/QueueingConsumer.java @@ -119,7 +119,7 @@ public void handleDelivery( String consumerTag, Envelope envelope, AMQP.BasicProperties properties, byte[] body) throws IOException { checkShutdown(); - log.info(new String(body) + "?????????"); + log.info(new String(body)); handover.produce(new Delivery(envelope, properties, body)); } } diff --git a/seatunnel-connectors-v2/connector-rabbitmq/src/main/java/org/apache/seatunnel/connectors/seatunnel/rabbitmq/source/RabbitmqSource.java b/seatunnel-connectors-v2/connector-rabbitmq/src/main/java/org/apache/seatunnel/connectors/seatunnel/rabbitmq/source/RabbitmqSource.java index 5d430219759..230b1ab84a0 100644 --- a/seatunnel-connectors-v2/connector-rabbitmq/src/main/java/org/apache/seatunnel/connectors/seatunnel/rabbitmq/source/RabbitmqSource.java +++ b/seatunnel-connectors-v2/connector-rabbitmq/src/main/java/org/apache/seatunnel/connectors/seatunnel/rabbitmq/source/RabbitmqSource.java @@ -27,6 +27,7 @@ import org.apache.seatunnel.api.table.type.SeaTunnelDataType; import org.apache.seatunnel.api.table.type.SeaTunnelRow; import org.apache.seatunnel.api.table.type.SeaTunnelRowType; +import org.apache.seatunnel.common.constants.JobMode; import org.apache.seatunnel.connectors.seatunnel.common.schema.SeaTunnelSchema; import org.apache.seatunnel.connectors.seatunnel.rabbitmq.config.RabbitmqConfig; import org.apache.seatunnel.connectors.seatunnel.rabbitmq.split.RabbitmqSplit; @@ -46,7 +47,7 @@ public class RabbitmqSource implements SeaTunnelSource(); - private static final Tuple2> TEST_DATASET = generateTestDataSet(); private GenericContainer rabbitmqContainer; @@ -81,6 +80,7 @@ public class RabbitmqIT extends TestSuiteBase implements TestResource { Channel sinkChannel = null; RabbitmqClient rabbitmqClient; RabbitmqClient sinkRabbitmqClient; + Thread thread; @BeforeAll @Override @@ -102,9 +102,18 @@ public void startUp() throws Exception { private void initSourceData() throws IOException, InterruptedException { JsonSerializationSchema jsonSerializationSchema = new JsonSerializationSchema(TEST_DATASET._1()); List rows = TEST_DATASET._2(); - for (int i = 0; i < rows.size(); i++) { - rabbitmqClient.write(new String(jsonSerializationSchema.serialize(rows.get(i))).getBytes(StandardCharsets.UTF_8)); - } + +// thread = new Thread(() -> { +// for (int i = 0; i < rows.size(); i++) { +// rabbitmqClient.write(new String(jsonSerializationSchema.serialize(rows.get(1))).getBytes(StandardCharsets.UTF_8)); +// try { +// Thread.sleep(10); +// } catch (InterruptedException e) { +// e.printStackTrace(); +// } +// } +// }); +// thread.start(); } private static Tuple2> generateTestDataSet() { @@ -207,9 +216,6 @@ private void initSinkRabbitMQ() { @AfterAll @Override public void tearDown() throws Exception { - if (channel != null) { - channel.close(); - } if (connection != null) { connection.close(); } @@ -218,26 +224,35 @@ public void tearDown() throws Exception { @TestTemplate public void testRabbitMQ(TestContainer container) throws Exception { + JsonSerializationSchema jsonSerializationSchema = new JsonSerializationSchema(TEST_DATASET._1()); + List rows = TEST_DATASET._2(); + for (int i = 0; i < rows.size(); i++) { + rabbitmqClient.write(new String(jsonSerializationSchema.serialize(rows.get(1))).getBytes(StandardCharsets.UTF_8)); + } + Thread.sleep(5); Container.ExecResult execResult = container.executeJob("/rabbitmq-to-rabbitmq.conf"); Assertions.assertEquals(0, execResult.getExitCode()); - RabbitmqConfig config = new RabbitmqConfig(); - config.setHost(rabbitmqContainer.getHost()); - config.setPort(rabbitmqContainer.getFirstMappedPort()); - config.setQueueName("test1"); - config.setVirtualHost("/"); - config.setUsername("guest"); - config.setPassword("guest"); - - DefaultConsumer consumer = sinkRabbitmqClient.getQueueingConsumer(handover); - sinkChannel.basicConsume(config.getQueueName(), true, consumer); - for(int i = 0; i < 100; i++) { - Optional deliveryOptional = handover.pollNext(); - if (deliveryOptional.isPresent()) { - Delivery delivery = deliveryOptional.get(); - byte[] body = delivery.getBody(); - System.out.println(new String(body)); - } - } + +// RabbitmqConfig config = new RabbitmqConfig(); +// config.setHost(rabbitmqContainer.getHost()); +// config.setPort(rabbitmqContainer.getFirstMappedPort()); +// config.setQueueName("test1"); +// config.setVirtualHost("/"); +// config.setUsername("guest"); +// config.setPassword("guest"); +// Set sets = new HashSet<>(); +// DefaultConsumer consumer = sinkRabbitmqClient.getQueueingConsumer(handover); +// sinkChannel.basicConsume(config.getQueueName(), true, consumer); +// for (int i = 0; i < 5; i++) { +// Optional deliveryOptional = handover.pollNext(); +// if (deliveryOptional.isPresent()) { +// Delivery delivery = deliveryOptional.get(); +// byte[] body = delivery.getBody(); +// sets.add(new String(body)); +// } +// } +// Assertions.assertTrue(sets.size() > 0); + } } diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-rabbitmq-e2e/src/test/resources/rabbitmq-to-rabbitmq.conf b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-rabbitmq-e2e/src/test/resources/rabbitmq-to-rabbitmq.conf index 0d5818f6465..da2ea00320a 100644 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-rabbitmq-e2e/src/test/resources/rabbitmq-to-rabbitmq.conf +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-rabbitmq-e2e/src/test/resources/rabbitmq-to-rabbitmq.conf @@ -17,6 +17,7 @@ env { execution.parallelism = 1 + job.mode = "BATCH" } source { @@ -54,7 +55,6 @@ transform { } sink { - Console {} RabbitMQ { host = "rabbitmq-e2e" port = 5672 From 750fdf108523f678c3254f6523bd7691d4053a38 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=AF=95=E5=8D=9A?= Date: Sun, 6 Nov 2022 16:04:20 +0800 Subject: [PATCH 06/15] [Feature][Connector-V2] add rabbitmq connector --- docs/en/connector-v2/sink/Rabbitmq.md | 95 +++++++++++ docs/en/connector-v2/source/Rabbitmq.md | 148 ++++++++++++++++++ .../connector-rabbitmq/pom.xml | 5 - .../rabbitmq/client/QueueingConsumer.java | 36 ----- .../rabbitmq/client/RabbitmqClient.java | 4 + .../rabbitmq/config/RabbitmqConfig.java | 11 +- .../rabbitmq/source/DeliveryMessage.java | 1 - .../rabbitmq/source/RabbitmqSourceReader.java | 3 +- .../source/RabbitmqSplitEnumerator.java | 14 +- .../e2e/connector/rabbitmq/RabbitmqIT.java | 111 ++++++------- .../test/resources/rabbitmq-to-rabbitmq.conf | 37 +++-- .../src/test/resources/log4j.properties | 22 --- 12 files changed, 329 insertions(+), 158 deletions(-) create mode 100644 docs/en/connector-v2/sink/Rabbitmq.md create mode 100644 docs/en/connector-v2/source/Rabbitmq.md delete mode 100644 seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/src/test/resources/log4j.properties diff --git a/docs/en/connector-v2/sink/Rabbitmq.md b/docs/en/connector-v2/sink/Rabbitmq.md new file mode 100644 index 00000000000..28931e99f62 --- /dev/null +++ b/docs/en/connector-v2/sink/Rabbitmq.md @@ -0,0 +1,95 @@ +# Rabbitmq + +> Rabbitmq sink connector + +## Description + +Used to write data to Rabbitmq. + +## Key features + +- [ ] [exactly-once](../../concept/connector-v2-features.md) +- [ ] [schema projection](../../concept/connector-v2-features.md) + +## Options + +| name | type | required | default value | +|-----------------------------|---------|----------|---------------| +| host | string | yes | - | +| port | int | yes | - | +| virtual_host | string | yes | - | +| username | string | yes | - | +| password | string | yes | - | +| url | string | no | - | +| queue_name | string | yes | - | +| network_recovery_interval | int | no | - | +| topology_recovery_enabled | boolean | no | - | +| automatic_recovery_enabled | boolean | no | - | +| connection_timeout | int | no | - | + +### host [string] + +the default host to use for connections + +### port [int] + +the default port to use for connections + +### virtual_host [string] + +virtual host – the virtual host to use when connecting to the broker + +### username [string] + +the AMQP user name to use when connecting to the broker + +### password [string] + +the password to use when connecting to the broker + +### url [string] + +convenience method for setting the fields in an AMQP URI: host, port, username, password and virtual host + +### queue_name [string] + +the queue to write the message to + +### schema [Config] + +#### fields [Config] + +the schema fields of upstream data. + +### network_recovery_interval [string] + +how long will automatic recovery wait before attempting to reconnect, in ms + +### topology_recovery [string] + +if true, enables topology recovery + +### automatic_recovery [string] + +if true, enables connection recovery + +### connection_timeout [int] + +connection TCP establishment timeout in milliseconds; zero for infinite + +## Example + +simple: + +```hocon +sink { + RabbitMQ { + host = "rabbitmq-e2e" + port = 5672 + virtual_host = "/" + username = "guest" + password = "guest" + queue_name = "test1" + } +} +``` diff --git a/docs/en/connector-v2/source/Rabbitmq.md b/docs/en/connector-v2/source/Rabbitmq.md new file mode 100644 index 00000000000..84020b101ae --- /dev/null +++ b/docs/en/connector-v2/source/Rabbitmq.md @@ -0,0 +1,148 @@ +# Rabbitmq + +> Rabbitmq source connector + +## Description + +Used to read data from Rabbitmq. + +## Key features + +- [ ] [batch](../../concept/connector-v2-features.md) +- [x] [stream](../../concept/connector-v2-features.md) +- [x] [exactly-once](../../concept/connector-v2-features.md) +- [x] [schema projection](../../concept/connector-v2-features.md) +- [ ] [parallelism](../../concept/connector-v2-features.md) +- [ ] [support user-defined split](../../concept/connector-v2-features.md) + +:::tip +The source must be non-parallel (parallelism set to 1) in order to achieve exactly-once. This limitation is mainly due to RabbitMQ’s approach to dispatching messages from a single queue to multiple consumers. + +## Options + +| name | type | required | default value | +|-----------------------------|---------|----------|---------------| +| host | string | yes | - | +| port | int | yes | - | +| virtual_host | string | yes | - | +| username | string | yes | - | +| password | string | yes | - | +| url | string | no | - | +| queue_name | string | yes | - | +| routing_key | string | no | - | +| exchange | string | no | - | +| schema | config | yes | - | +| network_recovery_interval | int | no | - | +| topology_recovery_enabled | boolean | no | - | +| automatic_recovery_enabled | boolean | no | - | +| connection_timeout | int | no | - | +| requested_channel_max | int | no | - | +| requested_frame_max | int | no | - | +| requested_heartbeat | int | no | - | +| prefetch_count | int | no | - | +| delivery_timeout | long | no | - | + + +### host [string] + +the default host to use for connections + +### port [int] + +the default port to use for connections + +### virtual_host [string] + +virtual host – the virtual host to use when connecting to the broker + +### username [string] + +the AMQP user name to use when connecting to the broker + +### password [string] + +the password to use when connecting to the broker + +### url [string] + +convenience method for setting the fields in an AMQP URI: host, port, username, password and virtual host + +### queue_name [string] + +the queue to publish the message to + +### routing_key [string] + +the routing key to publish the message to + +### exchange [string] + +the exchange to publish the message to + +### schema [Config] + +#### fields [Config] + +the schema fields of upstream data. + +### network_recovery_interval [string] + +how long will automatic recovery wait before attempting to reconnect, in ms + +### topology_recovery [string] + +if true, enables topology recovery + +### automatic_recovery [string] + +if true, enables connection recovery + +### connection_timeout [int] + +connection tcp establishment timeout in milliseconds; zero for infinite + +### requested_channel_max [int] + +initially requested maximum channel number; zero for unlimited +**Note: Note the value must be between 0 and 65535 (unsigned short in AMQP 0-9-1). + +### requested_frame_max [int] + +the requested maximum frame size + +### requested_heartbeat [int] + +Set the requested heartbeat timeout +**Note: Note the value must be between 0 and 65535 (unsigned short in AMQP 0-9-1). + +### prefetch_count [int] + +prefetchCount the max number of messages to receive without acknowledgement + +### delivery_timeout [long] + +deliveryTimeout maximum wait time, in milliseconds, for the next message delivery + +## Example + +simple: + +```hocon +source { + RabbitMQ { + host = "rabbitmq-e2e" + port = 5672 + virtual_host = "/" + username = "guest" + password = "guest" + queue_name = "test" + schema = { + fields { + id = bigint + c_map = "map" + c_array = "array" + } + } + } +} +``` diff --git a/seatunnel-connectors-v2/connector-rabbitmq/pom.xml b/seatunnel-connectors-v2/connector-rabbitmq/pom.xml index d10ea57ba9a..c198a54a4d2 100644 --- a/seatunnel-connectors-v2/connector-rabbitmq/pom.xml +++ b/seatunnel-connectors-v2/connector-rabbitmq/pom.xml @@ -48,10 +48,5 @@ seatunnel-format-json ${project.version} - - org.apache.seatunnel - seatunnel-format-text - ${project.version} - diff --git a/seatunnel-connectors-v2/connector-rabbitmq/src/main/java/org/apache/seatunnel/connectors/seatunnel/rabbitmq/client/QueueingConsumer.java b/seatunnel-connectors-v2/connector-rabbitmq/src/main/java/org/apache/seatunnel/connectors/seatunnel/rabbitmq/client/QueueingConsumer.java index 36a27cce5be..7ae96f42394 100644 --- a/seatunnel-connectors-v2/connector-rabbitmq/src/main/java/org/apache/seatunnel/connectors/seatunnel/rabbitmq/client/QueueingConsumer.java +++ b/seatunnel-connectors-v2/connector-rabbitmq/src/main/java/org/apache/seatunnel/connectors/seatunnel/rabbitmq/client/QueueingConsumer.java @@ -31,7 +31,6 @@ import lombok.extern.slf4j.Slf4j; import java.io.IOException; -import java.util.concurrent.TimeUnit; @Slf4j public class QueueingConsumer extends DefaultConsumer { @@ -59,41 +58,6 @@ private void checkShutdown() { } } - private Delivery handle(Delivery delivery) throws Handover.ClosedException, InterruptedException { - if (delivery == POISON || delivery == null && (shutdown != null || cancelled != null)) { - if (delivery == POISON) { - handover.produce(POISON); - if (shutdown == null && cancelled == null) { - throw new IllegalStateException( - "POISON in queue, but null shutdown and null cancelled. " - + "This should never happen, please report as a BUG"); - } - } - if (null != shutdown) { - throw Utility.fixStackTrace(shutdown); - } - if (null != cancelled) { - throw Utility.fixStackTrace(cancelled); - } - } - return delivery; - } - - public Delivery nextDelivery() - throws Exception { - return handle(handover.pollNext().get()); - } - - public Delivery nextDelivery(long timeout) - throws Exception { - return nextDelivery(timeout, TimeUnit.MILLISECONDS); - } - - public Delivery nextDelivery(long timeout, TimeUnit unit) - throws Exception { - return handle(handover.pollNext().get()); - } - @Override public void handleShutdownSignal(String consumerTag, ShutdownSignalException sig) { shutdown = sig; diff --git a/seatunnel-connectors-v2/connector-rabbitmq/src/main/java/org/apache/seatunnel/connectors/seatunnel/rabbitmq/client/RabbitmqClient.java b/seatunnel-connectors-v2/connector-rabbitmq/src/main/java/org/apache/seatunnel/connectors/seatunnel/rabbitmq/client/RabbitmqClient.java index 4896f7ec1e3..c39b7ade6c5 100644 --- a/seatunnel-connectors-v2/connector-rabbitmq/src/main/java/org/apache/seatunnel/connectors/seatunnel/rabbitmq/client/RabbitmqClient.java +++ b/seatunnel-connectors-v2/connector-rabbitmq/src/main/java/org/apache/seatunnel/connectors/seatunnel/rabbitmq/client/RabbitmqClient.java @@ -49,6 +49,10 @@ public RabbitmqClient(RabbitmqConfig config) { this.connectionFactory = getConnectionFactory(); this.connection = connectionFactory.newConnection(); this.channel = connection.createChannel(); + //set channel prefetch count + if (config.getPrefetchCount() != null) { + channel.basicQos(config.getPrefetchCount(), true); + } setupQueue(); } catch (Exception e) { throw new RuntimeException( diff --git a/seatunnel-connectors-v2/connector-rabbitmq/src/main/java/org/apache/seatunnel/connectors/seatunnel/rabbitmq/config/RabbitmqConfig.java b/seatunnel-connectors-v2/connector-rabbitmq/src/main/java/org/apache/seatunnel/connectors/seatunnel/rabbitmq/config/RabbitmqConfig.java index fd36c01aba0..193222d8210 100644 --- a/seatunnel-connectors-v2/connector-rabbitmq/src/main/java/org/apache/seatunnel/connectors/seatunnel/rabbitmq/config/RabbitmqConfig.java +++ b/seatunnel-connectors-v2/connector-rabbitmq/src/main/java/org/apache/seatunnel/connectors/seatunnel/rabbitmq/config/RabbitmqConfig.java @@ -24,8 +24,8 @@ public class RabbitmqConfig implements Serializable { public static final String PASSWORD = "password"; public static final String URL = "url"; public static final String NETWORK_RECOVERY_INTERVAL = "network_recovery_interval"; - public static final String AUTOMATIC_RECOVERY = "automatic_recovery"; - public static final String TOPOLOGY_RECOVERY = "topology_recovery"; + public static final String AUTOMATIC_RECOVERY_ENABLED = "automatic_recovery_enabled"; + public static final String TOPOLOGY_RECOVERY_ENABLED = "topology_recovery_enabled"; public static final String CONNECTION_TIMEOUT = "connection_timeout"; public static final String REQUESTED_CHANNEL_MAX = "requested_channel_max"; public static final String REQUESTED_FRAME_MAX = "requested_frame_max"; @@ -88,8 +88,11 @@ public RabbitmqConfig(Config config) { if (config.hasPath(NETWORK_RECOVERY_INTERVAL)) { this.networkRecoveryInterval = config.getInt(NETWORK_RECOVERY_INTERVAL); } - if (config.hasPath(AUTOMATIC_RECOVERY)) { - this.automaticRecovery = config.getBoolean(AUTOMATIC_RECOVERY); + if (config.hasPath(AUTOMATIC_RECOVERY_ENABLED)) { + this.automaticRecovery = config.getBoolean(AUTOMATIC_RECOVERY_ENABLED); + } + if (config.hasPath(TOPOLOGY_RECOVERY_ENABLED)) { + this.topologyRecovery = config.getBoolean(TOPOLOGY_RECOVERY_ENABLED); } if (config.hasPath(CONNECTION_TIMEOUT)) { this.connectionTimeout = config.getInt(CONNECTION_TIMEOUT); diff --git a/seatunnel-connectors-v2/connector-rabbitmq/src/main/java/org/apache/seatunnel/connectors/seatunnel/rabbitmq/source/DeliveryMessage.java b/seatunnel-connectors-v2/connector-rabbitmq/src/main/java/org/apache/seatunnel/connectors/seatunnel/rabbitmq/source/DeliveryMessage.java index e3589dd35d5..7e8ec4e87d5 100644 --- a/seatunnel-connectors-v2/connector-rabbitmq/src/main/java/org/apache/seatunnel/connectors/seatunnel/rabbitmq/source/DeliveryMessage.java +++ b/seatunnel-connectors-v2/connector-rabbitmq/src/main/java/org/apache/seatunnel/connectors/seatunnel/rabbitmq/source/DeliveryMessage.java @@ -27,5 +27,4 @@ @Getter public final class DeliveryMessage { private final Delivery delivery; - private final String splitId; } diff --git a/seatunnel-connectors-v2/connector-rabbitmq/src/main/java/org/apache/seatunnel/connectors/seatunnel/rabbitmq/source/RabbitmqSourceReader.java b/seatunnel-connectors-v2/connector-rabbitmq/src/main/java/org/apache/seatunnel/connectors/seatunnel/rabbitmq/source/RabbitmqSourceReader.java index 3494cdcfbe8..0c91684032e 100644 --- a/seatunnel-connectors-v2/connector-rabbitmq/src/main/java/org/apache/seatunnel/connectors/seatunnel/rabbitmq/source/RabbitmqSourceReader.java +++ b/seatunnel-connectors-v2/connector-rabbitmq/src/main/java/org/apache/seatunnel/connectors/seatunnel/rabbitmq/source/RabbitmqSourceReader.java @@ -151,11 +151,12 @@ public List snapshotState(long checkpointId) throws Exception { @Override public void addSplits(List splits) { - + //do nothing } @Override public void handleNoMoreSplits() { + //do nothing } @Override diff --git a/seatunnel-connectors-v2/connector-rabbitmq/src/main/java/org/apache/seatunnel/connectors/seatunnel/rabbitmq/source/RabbitmqSplitEnumerator.java b/seatunnel-connectors-v2/connector-rabbitmq/src/main/java/org/apache/seatunnel/connectors/seatunnel/rabbitmq/source/RabbitmqSplitEnumerator.java index b912f4e4109..c01436830d7 100644 --- a/seatunnel-connectors-v2/connector-rabbitmq/src/main/java/org/apache/seatunnel/connectors/seatunnel/rabbitmq/source/RabbitmqSplitEnumerator.java +++ b/seatunnel-connectors-v2/connector-rabbitmq/src/main/java/org/apache/seatunnel/connectors/seatunnel/rabbitmq/source/RabbitmqSplitEnumerator.java @@ -26,22 +26,22 @@ public class RabbitmqSplitEnumerator implements SourceSplitEnumerator { @Override public void open() { - + //do nothing } @Override public void run() throws Exception { - + //do nothing } @Override public void close() throws IOException { - + //do nothing } @Override public void addSplitsBack(List splits, int subtaskId) { - + //do nothing } @Override @@ -51,12 +51,12 @@ public int currentUnassignedSplitSize() { @Override public void handleSplitRequest(int subtaskId) { - + //do nothing } @Override public void registerReader(int subtaskId) { - + //do nothing } @Override @@ -66,6 +66,6 @@ public Object snapshotState(long checkpointId) throws Exception { @Override public void notifyCheckpointComplete(long checkpointId) throws Exception { - // nothing + //do nothing } } diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-rabbitmq-e2e/src/test/java/org/apache/seatunnel/e2e/connector/rabbitmq/RabbitmqIT.java b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-rabbitmq-e2e/src/test/java/org/apache/seatunnel/e2e/connector/rabbitmq/RabbitmqIT.java index da99ec966f3..51709411818 100644 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-rabbitmq-e2e/src/test/java/org/apache/seatunnel/e2e/connector/rabbitmq/RabbitmqIT.java +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-rabbitmq-e2e/src/test/java/org/apache/seatunnel/e2e/connector/rabbitmq/RabbitmqIT.java @@ -34,7 +34,6 @@ import org.apache.seatunnel.e2e.common.container.TestContainer; import org.apache.seatunnel.format.json.JsonSerializationSchema; -import com.rabbitmq.client.Channel; import com.rabbitmq.client.Connection; import com.rabbitmq.client.DefaultConsumer; import com.rabbitmq.client.Delivery; @@ -57,7 +56,12 @@ import java.time.Duration; import java.time.LocalDate; import java.time.LocalDateTime; -import java.util.*; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashSet; +import java.util.List; +import java.util.Optional; +import java.util.Set; import java.util.stream.Stream; import scala.Tuple2; @@ -68,19 +72,16 @@ public class RabbitmqIT extends TestSuiteBase implements TestResource { private static final String HOST = "rabbitmq-e2e"; private static final int PORT = 5672; private static final String QUEUE_NAME = "test"; + private static final String SINK_QUEUE_NAME = "test1"; private static final String USERNAME = "guest"; private static final String PASSWORD = "guest"; - Handover handover = new Handover<>(); private static final Tuple2> TEST_DATASET = generateTestDataSet(); + private static final JsonSerializationSchema JSON_SERIALIZATION_SCHEMA = new JsonSerializationSchema(TEST_DATASET._1()); private GenericContainer rabbitmqContainer; - Connection connection = null; - Channel channel = null; - Channel sinkChannel = null; + Connection connection; RabbitmqClient rabbitmqClient; - RabbitmqClient sinkRabbitmqClient; - Thread thread; @BeforeAll @Override @@ -95,25 +96,13 @@ public void startUp() throws Exception { Startables.deepStart(Stream.of(rabbitmqContainer)).join(); log.info("rabbitmq container started"); this.initRabbitMQ(); - this.initSourceData(); - this.initSinkRabbitMQ(); } private void initSourceData() throws IOException, InterruptedException { - JsonSerializationSchema jsonSerializationSchema = new JsonSerializationSchema(TEST_DATASET._1()); List rows = TEST_DATASET._2(); - -// thread = new Thread(() -> { -// for (int i = 0; i < rows.size(); i++) { -// rabbitmqClient.write(new String(jsonSerializationSchema.serialize(rows.get(1))).getBytes(StandardCharsets.UTF_8)); -// try { -// Thread.sleep(10); -// } catch (InterruptedException e) { -// e.printStackTrace(); -// } -// } -// }); -// thread.start(); + for (int i = 0; i < rows.size(); i++) { + rabbitmqClient.write(new String(JSON_SERIALIZATION_SCHEMA.serialize(rows.get(1))).getBytes(StandardCharsets.UTF_8)); + } } private static Tuple2> generateTestDataSet() { @@ -156,10 +145,10 @@ private static Tuple2> generateTestDataSet( ); List rows = new ArrayList<>(); - for (int i = 0; i < 100; i++) { + for (int i = 0; i < 10; i++) { SeaTunnelRow row = new SeaTunnelRow( new Object[]{ - Long.valueOf(i), + Long.valueOf(1), Collections.singletonMap("key", Short.parseShort("1")), new Byte[]{Byte.parseByte("1")}, "string", @@ -181,33 +170,31 @@ private static Tuple2> generateTestDataSet( } private void initRabbitMQ() { - try { RabbitmqConfig config = new RabbitmqConfig(); config.setHost(rabbitmqContainer.getHost()); config.setPort(rabbitmqContainer.getFirstMappedPort()); - config.setQueueName("test"); + config.setQueueName(QUEUE_NAME); config.setVirtualHost("/"); - config.setUsername("guest"); - config.setPassword("guest"); + config.setUsername(USERNAME); + config.setPassword(PASSWORD); rabbitmqClient = new RabbitmqClient(config); } catch (Exception e) { throw new RuntimeException("init Rabbitmq error", e); } } - private void initSinkRabbitMQ() { + private RabbitmqClient initSinkRabbitMQ() { try { RabbitmqConfig config = new RabbitmqConfig(); config.setHost(rabbitmqContainer.getHost()); config.setPort(rabbitmqContainer.getFirstMappedPort()); - config.setQueueName("test1"); + config.setQueueName(SINK_QUEUE_NAME); config.setVirtualHost("/"); - config.setUsername("guest"); - config.setPassword("guest"); - sinkRabbitmqClient = new RabbitmqClient(config); - sinkChannel = sinkRabbitmqClient.getChannel(); + config.setUsername(USERNAME); + config.setPassword(PASSWORD); + return new RabbitmqClient(config); } catch (Exception e) { throw new RuntimeException("init Rabbitmq error", e); } @@ -224,35 +211,33 @@ public void tearDown() throws Exception { @TestTemplate public void testRabbitMQ(TestContainer container) throws Exception { - JsonSerializationSchema jsonSerializationSchema = new JsonSerializationSchema(TEST_DATASET._1()); - List rows = TEST_DATASET._2(); - for (int i = 0; i < rows.size(); i++) { - rabbitmqClient.write(new String(jsonSerializationSchema.serialize(rows.get(1))).getBytes(StandardCharsets.UTF_8)); - } - Thread.sleep(5); - Container.ExecResult execResult = container.executeJob("/rabbitmq-to-rabbitmq.conf"); - Assertions.assertEquals(0, execResult.getExitCode()); - -// RabbitmqConfig config = new RabbitmqConfig(); -// config.setHost(rabbitmqContainer.getHost()); -// config.setPort(rabbitmqContainer.getFirstMappedPort()); -// config.setQueueName("test1"); -// config.setVirtualHost("/"); -// config.setUsername("guest"); -// config.setPassword("guest"); -// Set sets = new HashSet<>(); -// DefaultConsumer consumer = sinkRabbitmqClient.getQueueingConsumer(handover); -// sinkChannel.basicConsume(config.getQueueName(), true, consumer); -// for (int i = 0; i < 5; i++) { -// Optional deliveryOptional = handover.pollNext(); -// if (deliveryOptional.isPresent()) { -// Delivery delivery = deliveryOptional.get(); -// byte[] body = delivery.getBody(); -// sets.add(new String(body)); -// } -// } -// Assertions.assertTrue(sets.size() > 0); + //send data to source queue before executeJob start in every testContainer + initSourceData(); + //init consumer client before executeJob start in every testContainer + RabbitmqClient sinkRabbitmqClient = initSinkRabbitMQ(); + Set resultSet = new HashSet<>(); + Handover handover = new Handover<>(); + DefaultConsumer consumer = sinkRabbitmqClient.getQueueingConsumer(handover); + sinkRabbitmqClient.getChannel().basicConsume(SINK_QUEUE_NAME, true, consumer); + // assert execute Job code + Container.ExecResult execResult = container.executeJob("/rabbitmq-to-rabbitmq.conf"); + Assertions.assertEquals(0, execResult.getExitCode()); + //consume data when every testContainer finished + //try to poll five times + for (int i = 0; i < 5; i++) { + Optional deliveryOptional = handover.pollNext(); + if (deliveryOptional.isPresent()) { + Delivery delivery = deliveryOptional.get(); + byte[] body = delivery.getBody(); + resultSet.add(new String(body)); + } + } + // close to prevent rabbitmq client consumer in the next TestContainer to consume + sinkRabbitmqClient.close(); + //assert source and sink data + Assertions.assertTrue(resultSet.size() > 0); + Assertions.assertTrue(resultSet.stream().findAny().get().equals(new String(JSON_SERIALIZATION_SCHEMA.serialize(TEST_DATASET._2().get(1))))); } } diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-rabbitmq-e2e/src/test/resources/rabbitmq-to-rabbitmq.conf b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-rabbitmq-e2e/src/test/resources/rabbitmq-to-rabbitmq.conf index da2ea00320a..2df6111f2d9 100644 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-rabbitmq-e2e/src/test/resources/rabbitmq-to-rabbitmq.conf +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-rabbitmq-e2e/src/test/resources/rabbitmq-to-rabbitmq.conf @@ -28,26 +28,25 @@ source { username = "guest" password = "guest" queue_name = "test" - schema = { - fields { - id = bigint - c_map = "map" - c_array = "array" - c_string = string - c_boolean = boolean - c_tinyint = tinyint - c_smallint = smallint - c_int = int - c_bigint = bigint - c_float = float - c_double = double - c_decimal = "decimal(2, 1)" - c_bytes = bytes - c_date = date - c_timestamp = timestamp - } + schema = { + fields { + id = bigint + c_map = "map" + c_array = "array" + c_string = string + c_boolean = boolean + c_tinyint = tinyint + c_smallint = smallint + c_int = int + c_bigint = bigint + c_float = float + c_double = double + c_decimal = "decimal(2, 1)" + c_bytes = bytes + c_date = date + c_timestamp = timestamp } - format = "json" + } } } diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/src/test/resources/log4j.properties b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/src/test/resources/log4j.properties deleted file mode 100644 index db5d9e51220..00000000000 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-redis-e2e/src/test/resources/log4j.properties +++ /dev/null @@ -1,22 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# Set everything to be logged to the console -log4j.rootCategory=INFO, console -log4j.appender.console=org.apache.log4j.ConsoleAppender -log4j.appender.console.target=System.err -log4j.appender.console.layout=org.apache.log4j.PatternLayout -log4j.appender.console.layout.ConversionPattern=%d{yy/MM/dd HH:mm:ss} %p %c{1}: %m%n From 2da9571c7db4e718068414854ee848f8694e0d5e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=AF=95=E5=8D=9A?= Date: Sun, 6 Nov 2022 16:22:29 +0800 Subject: [PATCH 07/15] fix code style --- .../rabbitmq/config/RabbitmqConfig.java | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/seatunnel-connectors-v2/connector-rabbitmq/src/main/java/org/apache/seatunnel/connectors/seatunnel/rabbitmq/config/RabbitmqConfig.java b/seatunnel-connectors-v2/connector-rabbitmq/src/main/java/org/apache/seatunnel/connectors/seatunnel/rabbitmq/config/RabbitmqConfig.java index 193222d8210..bb7fd99dd62 100644 --- a/seatunnel-connectors-v2/connector-rabbitmq/src/main/java/org/apache/seatunnel/connectors/seatunnel/rabbitmq/config/RabbitmqConfig.java +++ b/seatunnel-connectors-v2/connector-rabbitmq/src/main/java/org/apache/seatunnel/connectors/seatunnel/rabbitmq/config/RabbitmqConfig.java @@ -1,3 +1,20 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package org.apache.seatunnel.connectors.seatunnel.rabbitmq.config; import org.apache.seatunnel.common.config.TypesafeConfigUtils; From 8c1f3275c17ea00d006aa0fe57a1562eccb20fba Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=AF=95=E5=8D=9A?= Date: Sun, 6 Nov 2022 18:36:12 +0800 Subject: [PATCH 08/15] fix code style --- docs/en/connector-v2/sink/Rabbitmq.md | 2 +- docs/en/connector-v2/source/Rabbitmq.md | 4 +-- .../rabbitmq/client/RabbitmqClient.java | 29 ++++--------------- .../seatunnel/rabbitmq/sink/RabbitmqSink.java | 15 ++++++++++ .../rabbitmq/sink/RabbitmqSinkWriter.java | 5 ---- .../rabbitmq/source/RabbitmqSource.java | 15 ++++++++++ .../rabbitmq/source/RabbitmqSourceReader.java | 1 + 7 files changed, 40 insertions(+), 31 deletions(-) diff --git a/docs/en/connector-v2/sink/Rabbitmq.md b/docs/en/connector-v2/sink/Rabbitmq.md index 28931e99f62..6210747b4e4 100644 --- a/docs/en/connector-v2/sink/Rabbitmq.md +++ b/docs/en/connector-v2/sink/Rabbitmq.md @@ -20,8 +20,8 @@ Used to write data to Rabbitmq. | virtual_host | string | yes | - | | username | string | yes | - | | password | string | yes | - | -| url | string | no | - | | queue_name | string | yes | - | +| url | string | no | - | | network_recovery_interval | int | no | - | | topology_recovery_enabled | boolean | no | - | | automatic_recovery_enabled | boolean | no | - | diff --git a/docs/en/connector-v2/source/Rabbitmq.md b/docs/en/connector-v2/source/Rabbitmq.md index 84020b101ae..3884f2bb9e5 100644 --- a/docs/en/connector-v2/source/Rabbitmq.md +++ b/docs/en/connector-v2/source/Rabbitmq.md @@ -27,11 +27,11 @@ The source must be non-parallel (parallelism set to 1) in order to achieve exact | virtual_host | string | yes | - | | username | string | yes | - | | password | string | yes | - | -| url | string | no | - | | queue_name | string | yes | - | +| schema | config | yes | - | +| url | string | no | - | | routing_key | string | no | - | | exchange | string | no | - | -| schema | config | yes | - | | network_recovery_interval | int | no | - | | topology_recovery_enabled | boolean | no | - | | automatic_recovery_enabled | boolean | no | - | diff --git a/seatunnel-connectors-v2/connector-rabbitmq/src/main/java/org/apache/seatunnel/connectors/seatunnel/rabbitmq/client/RabbitmqClient.java b/seatunnel-connectors-v2/connector-rabbitmq/src/main/java/org/apache/seatunnel/connectors/seatunnel/rabbitmq/client/RabbitmqClient.java index c39b7ade6c5..946bac4d871 100644 --- a/seatunnel-connectors-v2/connector-rabbitmq/src/main/java/org/apache/seatunnel/connectors/seatunnel/rabbitmq/client/RabbitmqClient.java +++ b/seatunnel-connectors-v2/connector-rabbitmq/src/main/java/org/apache/seatunnel/connectors/seatunnel/rabbitmq/client/RabbitmqClient.java @@ -55,12 +55,8 @@ public RabbitmqClient(RabbitmqConfig config) { } setupQueue(); } catch (Exception e) { - throw new RuntimeException( - "Error while create RMQ client with " - + this.config.getQueueName() - + " at " - + this.config.getHost(), - e); + throw new RuntimeException(String.format("Error while create RMQ client with %s at %s", config.getQueueName(), config.getHost()), e); + } } @@ -140,18 +136,9 @@ public void write(byte[] msg) { } } catch (IOException e) { if (config.isLogFailuresOnly()) { - log.error( - "Cannot send RMQ message {} at {}", - config.getQueueName(), - config.getHost(), - e); + log.error("Cannot send RMQ message {} at {}", config.getQueueName(), config.getHost(), e); } else { - throw new RuntimeException( - "Cannot send RMQ message " - + config.getQueueName() - + " at " - + config.getHost(), - e); + throw new RuntimeException(String.format("Cannot send RMQ message %s at %s", config.getQueueName(), config.getHost()), e); } } } @@ -179,12 +166,8 @@ public void close() { t = e; } if (t != null) { - throw new RuntimeException( - "Error while closing RMQ connection with " - + this.config.getQueueName() - + " at " - + this.config.getHost(), - t); + throw new RuntimeException(String.format("Error while closing RMQ connection with %s at %s", config.getQueueName(), config.getHost()), t); + } } diff --git a/seatunnel-connectors-v2/connector-rabbitmq/src/main/java/org/apache/seatunnel/connectors/seatunnel/rabbitmq/sink/RabbitmqSink.java b/seatunnel-connectors-v2/connector-rabbitmq/src/main/java/org/apache/seatunnel/connectors/seatunnel/rabbitmq/sink/RabbitmqSink.java index 66af5ea5339..d8489c71aef 100644 --- a/seatunnel-connectors-v2/connector-rabbitmq/src/main/java/org/apache/seatunnel/connectors/seatunnel/rabbitmq/sink/RabbitmqSink.java +++ b/seatunnel-connectors-v2/connector-rabbitmq/src/main/java/org/apache/seatunnel/connectors/seatunnel/rabbitmq/sink/RabbitmqSink.java @@ -17,12 +17,22 @@ package org.apache.seatunnel.connectors.seatunnel.rabbitmq.sink; +import static org.apache.seatunnel.connectors.seatunnel.rabbitmq.config.RabbitmqConfig.HOST; +import static org.apache.seatunnel.connectors.seatunnel.rabbitmq.config.RabbitmqConfig.PASSWORD; +import static org.apache.seatunnel.connectors.seatunnel.rabbitmq.config.RabbitmqConfig.PORT; +import static org.apache.seatunnel.connectors.seatunnel.rabbitmq.config.RabbitmqConfig.QUEUE_NAME; +import static org.apache.seatunnel.connectors.seatunnel.rabbitmq.config.RabbitmqConfig.USERNAME; +import static org.apache.seatunnel.connectors.seatunnel.rabbitmq.config.RabbitmqConfig.VIRTUAL_HOST; + import org.apache.seatunnel.api.common.PrepareFailException; import org.apache.seatunnel.api.sink.SeaTunnelSink; import org.apache.seatunnel.api.sink.SinkWriter; import org.apache.seatunnel.api.table.type.SeaTunnelDataType; import org.apache.seatunnel.api.table.type.SeaTunnelRow; import org.apache.seatunnel.api.table.type.SeaTunnelRowType; +import org.apache.seatunnel.common.config.CheckConfigUtil; +import org.apache.seatunnel.common.config.CheckResult; +import org.apache.seatunnel.common.constants.PluginType; import org.apache.seatunnel.connectors.seatunnel.common.sink.AbstractSimpleSink; import org.apache.seatunnel.connectors.seatunnel.common.sink.AbstractSinkWriter; import org.apache.seatunnel.connectors.seatunnel.rabbitmq.config.RabbitmqConfig; @@ -47,6 +57,11 @@ public String getPluginName() { @Override public void prepare(Config pluginConfig) throws PrepareFailException { this.pluginConfig = pluginConfig; + + CheckResult result = CheckConfigUtil.checkAllExists(pluginConfig, HOST, PORT, VIRTUAL_HOST, USERNAME, PASSWORD, QUEUE_NAME); + if (!result.isSuccess()) { + throw new PrepareFailException(getPluginName(), PluginType.SINK, result.getMsg()); + } rabbitMQConfig = new RabbitmqConfig(pluginConfig); } diff --git a/seatunnel-connectors-v2/connector-rabbitmq/src/main/java/org/apache/seatunnel/connectors/seatunnel/rabbitmq/sink/RabbitmqSinkWriter.java b/seatunnel-connectors-v2/connector-rabbitmq/src/main/java/org/apache/seatunnel/connectors/seatunnel/rabbitmq/sink/RabbitmqSinkWriter.java index 5be04b4d3ba..a5d7c06b5da 100644 --- a/seatunnel-connectors-v2/connector-rabbitmq/src/main/java/org/apache/seatunnel/connectors/seatunnel/rabbitmq/sink/RabbitmqSinkWriter.java +++ b/seatunnel-connectors-v2/connector-rabbitmq/src/main/java/org/apache/seatunnel/connectors/seatunnel/rabbitmq/sink/RabbitmqSinkWriter.java @@ -24,13 +24,9 @@ import org.apache.seatunnel.connectors.seatunnel.rabbitmq.config.RabbitmqConfig; import org.apache.seatunnel.format.json.JsonSerializationSchema; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - import java.util.Optional; public class RabbitmqSinkWriter extends AbstractSinkWriter { - private static final Logger LOG = LoggerFactory.getLogger(RabbitmqSinkWriter.class); private RabbitmqClient rabbitMQClient; private final JsonSerializationSchema jsonSerializationSchema; @@ -46,7 +42,6 @@ public void write(SeaTunnelRow element) { @Override public Optional prepareCommit() { - return Optional.empty(); } diff --git a/seatunnel-connectors-v2/connector-rabbitmq/src/main/java/org/apache/seatunnel/connectors/seatunnel/rabbitmq/source/RabbitmqSource.java b/seatunnel-connectors-v2/connector-rabbitmq/src/main/java/org/apache/seatunnel/connectors/seatunnel/rabbitmq/source/RabbitmqSource.java index 230b1ab84a0..6f980ee576c 100644 --- a/seatunnel-connectors-v2/connector-rabbitmq/src/main/java/org/apache/seatunnel/connectors/seatunnel/rabbitmq/source/RabbitmqSource.java +++ b/seatunnel-connectors-v2/connector-rabbitmq/src/main/java/org/apache/seatunnel/connectors/seatunnel/rabbitmq/source/RabbitmqSource.java @@ -17,6 +17,14 @@ package org.apache.seatunnel.connectors.seatunnel.rabbitmq.source; +import static org.apache.seatunnel.connectors.seatunnel.common.config.CommonConfig.SCHEMA; +import static org.apache.seatunnel.connectors.seatunnel.rabbitmq.config.RabbitmqConfig.HOST; +import static org.apache.seatunnel.connectors.seatunnel.rabbitmq.config.RabbitmqConfig.PASSWORD; +import static org.apache.seatunnel.connectors.seatunnel.rabbitmq.config.RabbitmqConfig.PORT; +import static org.apache.seatunnel.connectors.seatunnel.rabbitmq.config.RabbitmqConfig.QUEUE_NAME; +import static org.apache.seatunnel.connectors.seatunnel.rabbitmq.config.RabbitmqConfig.USERNAME; +import static org.apache.seatunnel.connectors.seatunnel.rabbitmq.config.RabbitmqConfig.VIRTUAL_HOST; + import org.apache.seatunnel.api.common.JobContext; import org.apache.seatunnel.api.common.PrepareFailException; import org.apache.seatunnel.api.serialization.DeserializationSchema; @@ -27,7 +35,10 @@ import org.apache.seatunnel.api.table.type.SeaTunnelDataType; import org.apache.seatunnel.api.table.type.SeaTunnelRow; import org.apache.seatunnel.api.table.type.SeaTunnelRowType; +import org.apache.seatunnel.common.config.CheckConfigUtil; +import org.apache.seatunnel.common.config.CheckResult; import org.apache.seatunnel.common.constants.JobMode; +import org.apache.seatunnel.common.constants.PluginType; import org.apache.seatunnel.connectors.seatunnel.common.schema.SeaTunnelSchema; import org.apache.seatunnel.connectors.seatunnel.rabbitmq.config.RabbitmqConfig; import org.apache.seatunnel.connectors.seatunnel.rabbitmq.split.RabbitmqSplit; @@ -57,6 +68,10 @@ public String getPluginName() { @Override public void prepare(Config config) throws PrepareFailException { + CheckResult result = CheckConfigUtil.checkAllExists(config, HOST, PORT, VIRTUAL_HOST, USERNAME, PASSWORD, QUEUE_NAME, SCHEMA); + if (!result.isSuccess()) { + throw new PrepareFailException(getPluginName(), PluginType.SINK, result.getMsg()); + } this.rabbitMQConfig = new RabbitmqConfig(config); setDeserialization(config); } diff --git a/seatunnel-connectors-v2/connector-rabbitmq/src/main/java/org/apache/seatunnel/connectors/seatunnel/rabbitmq/source/RabbitmqSourceReader.java b/seatunnel-connectors-v2/connector-rabbitmq/src/main/java/org/apache/seatunnel/connectors/seatunnel/rabbitmq/source/RabbitmqSourceReader.java index 0c91684032e..0f819796996 100644 --- a/seatunnel-connectors-v2/connector-rabbitmq/src/main/java/org/apache/seatunnel/connectors/seatunnel/rabbitmq/source/RabbitmqSourceReader.java +++ b/seatunnel-connectors-v2/connector-rabbitmq/src/main/java/org/apache/seatunnel/connectors/seatunnel/rabbitmq/source/RabbitmqSourceReader.java @@ -121,6 +121,7 @@ public void pollNext(Collector output) throws Exception { } if (Boundedness.BOUNDED.equals(context.getBoundedness())) { // signal to the source that we have reached the end of the data. + // rabbitmq source connector on support streaming mode, this is for test context.signalNoMoreElement(); } } From ada134005622bd3823e2d2785e06da857e6395a9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=AF=95=E5=8D=9A?= Date: Wed, 9 Nov 2022 23:11:47 +0800 Subject: [PATCH 09/15] merge dev --- docs/en/connector-v2/sink/Rabbitmq.md | 6 ++++++ docs/en/connector-v2/source/Rabbitmq.md | 6 ++++++ plugin-mapping.properties | 3 ++- seatunnel-connectors-v2/pom.xml | 1 + seatunnel-e2e/seatunnel-connector-v2-e2e/pom.xml | 1 + 5 files changed, 16 insertions(+), 1 deletion(-) diff --git a/docs/en/connector-v2/sink/Rabbitmq.md b/docs/en/connector-v2/sink/Rabbitmq.md index 6210747b4e4..f484602a14b 100644 --- a/docs/en/connector-v2/sink/Rabbitmq.md +++ b/docs/en/connector-v2/sink/Rabbitmq.md @@ -93,3 +93,9 @@ sink { } } ``` + +## Changelog + +### next version + +- Add Rabbitmq Sink Connector diff --git a/docs/en/connector-v2/source/Rabbitmq.md b/docs/en/connector-v2/source/Rabbitmq.md index 3884f2bb9e5..5ce560e9f36 100644 --- a/docs/en/connector-v2/source/Rabbitmq.md +++ b/docs/en/connector-v2/source/Rabbitmq.md @@ -146,3 +146,9 @@ source { } } ``` + +## Changelog + +### next version + +- Add Rabbitmq source Connector diff --git a/plugin-mapping.properties b/plugin-mapping.properties index 6863ef37e55..f72c1c56f27 100644 --- a/plugin-mapping.properties +++ b/plugin-mapping.properties @@ -144,4 +144,5 @@ seatunnel.sink.Cassandra = connector-cassandra seatunnel.sink.StarRocks = connector-starrocks seatunnel.source.MyHours = connector-http-myhours seatunnel.sink.InfluxDB = connector-influxdb -seatunnel.source.GoogleSheets = connector-google-sheets \ No newline at end of file +seatunnel.source.GoogleSheets = connector-google-sheets +seatunnel.sink.RabbitMQ = connector-rabbitmq \ No newline at end of file diff --git a/seatunnel-connectors-v2/pom.xml b/seatunnel-connectors-v2/pom.xml index 28ac5a803f9..31ea455f817 100644 --- a/seatunnel-connectors-v2/pom.xml +++ b/seatunnel-connectors-v2/pom.xml @@ -60,6 +60,7 @@ connector-cassandra connector-starrocks connector-google-sheets + connector-rabbitmq diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/pom.xml b/seatunnel-e2e/seatunnel-connector-v2-e2e/pom.xml index eb8614495cb..bcd66e05483 100644 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/pom.xml +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/pom.xml @@ -34,6 +34,7 @@ connector-file-local-e2e connector-cassandra-e2e connector-http-e2e + connector-rabbitmq-e2e seatunnel-connector-v2-e2e From 029aa7d1524f2e434af94f6e990a1bdb7320f5e6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=AF=95=E5=8D=9A?= Date: Fri, 11 Nov 2022 20:56:33 +0800 Subject: [PATCH 10/15] fix code style --- docs/en/connector-v2/source/Rabbitmq.md | 6 ++-- .../test/resources/rabbitmq-to-rabbitmq.conf | 32 +++++++++---------- 2 files changed, 19 insertions(+), 19 deletions(-) diff --git a/docs/en/connector-v2/source/Rabbitmq.md b/docs/en/connector-v2/source/Rabbitmq.md index 5ce560e9f36..3ba22ad632c 100644 --- a/docs/en/connector-v2/source/Rabbitmq.md +++ b/docs/en/connector-v2/source/Rabbitmq.md @@ -138,9 +138,9 @@ source { queue_name = "test" schema = { fields { - id = bigint - c_map = "map" - c_array = "array" + id = bigint + c_map = "map" + c_array = "array" } } } diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-rabbitmq-e2e/src/test/resources/rabbitmq-to-rabbitmq.conf b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-rabbitmq-e2e/src/test/resources/rabbitmq-to-rabbitmq.conf index 2df6111f2d9..9ad402ff74f 100644 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-rabbitmq-e2e/src/test/resources/rabbitmq-to-rabbitmq.conf +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-rabbitmq-e2e/src/test/resources/rabbitmq-to-rabbitmq.conf @@ -17,7 +17,7 @@ env { execution.parallelism = 1 - job.mode = "BATCH" + job.mode = "BATCH" } source { @@ -30,21 +30,21 @@ source { queue_name = "test" schema = { fields { - id = bigint - c_map = "map" - c_array = "array" - c_string = string - c_boolean = boolean - c_tinyint = tinyint - c_smallint = smallint - c_int = int - c_bigint = bigint - c_float = float - c_double = double - c_decimal = "decimal(2, 1)" - c_bytes = bytes - c_date = date - c_timestamp = timestamp + id = bigint + c_map = "map" + c_array = "array" + c_string = string + c_boolean = boolean + c_tinyint = tinyint + c_smallint = smallint + c_int = int + c_bigint = bigint + c_float = float + c_double = double + c_decimal = "decimal(2, 1)" + c_bytes = bytes + c_date = date + c_timestamp = timestamp } } } From c4064b2f69bc6915029c07d77a21dc6006cda474 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=AF=95=E5=8D=9A?= Date: Mon, 21 Nov 2022 17:42:34 +0800 Subject: [PATCH 11/15] add e2e switch in rabbitmq config --- .../seatunnel/rabbitmq/config/RabbitmqConfig.java | 8 +++++++- .../seatunnel/rabbitmq/source/RabbitmqSource.java | 5 ++++- .../src/test/resources/rabbitmq-to-rabbitmq.conf | 3 ++- 3 files changed, 13 insertions(+), 3 deletions(-) diff --git a/seatunnel-connectors-v2/connector-rabbitmq/src/main/java/org/apache/seatunnel/connectors/seatunnel/rabbitmq/config/RabbitmqConfig.java b/seatunnel-connectors-v2/connector-rabbitmq/src/main/java/org/apache/seatunnel/connectors/seatunnel/rabbitmq/config/RabbitmqConfig.java index bb7fd99dd62..ea2d7a8d3b8 100644 --- a/seatunnel-connectors-v2/connector-rabbitmq/src/main/java/org/apache/seatunnel/connectors/seatunnel/rabbitmq/config/RabbitmqConfig.java +++ b/seatunnel-connectors-v2/connector-rabbitmq/src/main/java/org/apache/seatunnel/connectors/seatunnel/rabbitmq/config/RabbitmqConfig.java @@ -54,8 +54,8 @@ public class RabbitmqConfig implements Serializable { public static final String ROUTING_KEY = "routing_key"; public static final String EXCHANGE = "exchange"; - public static final String LOG_FAILURES_ONLY = "log_failures_only"; + public static final String FOR_E2E_TESTING = "for_e2e_testing"; private String host; private Integer port; @@ -76,6 +76,9 @@ public class RabbitmqConfig implements Serializable { private String routingKey; private boolean logFailuresOnly = false; private String exchange = ""; + + private boolean forE2ETesting = false; + public static final String RABBITMQ_SINK_CONFIG_PREFIX = "rabbitmq.properties."; private final Map sinkOptionProps = new HashMap<>(); @@ -135,6 +138,9 @@ public RabbitmqConfig(Config config) { if (config.hasPath(EXCHANGE)) { this.exchange = config.getString(EXCHANGE); } + if (config.hasPath(FOR_E2E_TESTING)) { + this.forE2ETesting = config.getBoolean(FOR_E2E_TESTING); + } parseSinkOptionProperties(config); } diff --git a/seatunnel-connectors-v2/connector-rabbitmq/src/main/java/org/apache/seatunnel/connectors/seatunnel/rabbitmq/source/RabbitmqSource.java b/seatunnel-connectors-v2/connector-rabbitmq/src/main/java/org/apache/seatunnel/connectors/seatunnel/rabbitmq/source/RabbitmqSource.java index e8101e7f14a..6652ee6a486 100644 --- a/seatunnel-connectors-v2/connector-rabbitmq/src/main/java/org/apache/seatunnel/connectors/seatunnel/rabbitmq/source/RabbitmqSource.java +++ b/seatunnel-connectors-v2/connector-rabbitmq/src/main/java/org/apache/seatunnel/connectors/seatunnel/rabbitmq/source/RabbitmqSource.java @@ -58,7 +58,10 @@ public class RabbitmqSource implements SeaTunnelSource Date: Mon, 21 Nov 2022 18:28:46 +0800 Subject: [PATCH 12/15] improve RabbitmqSourceReader.pollNext IN E2E --- .../rabbitmq/source/RabbitmqSourceReader.java | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/seatunnel-connectors-v2/connector-rabbitmq/src/main/java/org/apache/seatunnel/connectors/seatunnel/rabbitmq/source/RabbitmqSourceReader.java b/seatunnel-connectors-v2/connector-rabbitmq/src/main/java/org/apache/seatunnel/connectors/seatunnel/rabbitmq/source/RabbitmqSourceReader.java index fc99b8d7603..f1df9263852 100644 --- a/seatunnel-connectors-v2/connector-rabbitmq/src/main/java/org/apache/seatunnel/connectors/seatunnel/rabbitmq/source/RabbitmqSourceReader.java +++ b/seatunnel-connectors-v2/connector-rabbitmq/src/main/java/org/apache/seatunnel/connectors/seatunnel/rabbitmq/source/RabbitmqSourceReader.java @@ -122,11 +122,12 @@ public void pollNext(Collector output) throws Exception { deliveryTagsProcessedForCurrentSnapshot.add(envelope.getDeliveryTag()); deserializationSchema.deserialize(body, output); } - } - if (Boundedness.BOUNDED.equals(context.getBoundedness())) { - // signal to the source that we have reached the end of the data. - // rabbitmq source connector on support streaming mode, this is for test - context.signalNoMoreElement(); + + if (Boundedness.BOUNDED.equals(context.getBoundedness())) { + // signal to the source that we have reached the end of the data. + // rabbitmq source connector on support streaming mode, this is for test + context.signalNoMoreElement(); + } } } From 0cd772330bb0707043e139aa147df375646782b1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=AF=95=E5=8D=9A?= Date: Tue, 22 Nov 2022 15:03:43 +0800 Subject: [PATCH 13/15] add Rabbitmq Connector Error Codes in doc --- .../Error-Quick-Reference-Manual.md | 14 ++++++++++ .../rabbitmq/client/RabbitmqClient.java | 26 +++++++------------ .../exception/RabbitmqConnectorErrorCode.java | 5 +++- 3 files changed, 28 insertions(+), 17 deletions(-) diff --git a/docs/en/connector-v2/Error-Quick-Reference-Manual.md b/docs/en/connector-v2/Error-Quick-Reference-Manual.md index e30979da596..5e8eb726e50 100644 --- a/docs/en/connector-v2/Error-Quick-Reference-Manual.md +++ b/docs/en/connector-v2/Error-Quick-Reference-Manual.md @@ -46,3 +46,17 @@ This document records some common error codes and corresponding solutions of Sea | CASSANDRA-03 | Close cql session of cassandra failed | When users encounter this error code, it means that cassandra has some problems, please check it whether is work | | CASSANDRA-04 | No data in source table | When users encounter this error code, it means that source cassandra table has no data, please check it | | CASSANDRA-05 | Parse ip address from string field field | When users encounter this error code, it means that upstream data does not match ip address format, please check it | + +## Rabbitmq Connector Error Codes + +| code | description | solution | +|-------------|---------------------------------------------------------------|-----------------------------------------------------------------------------------------------------------------| +| RABBITMQ-01 | handle queue consumer shutdown signal failed | When users encounter this error code, it means that job has some problems, please check it whether is work well | +| RABBITMQ-02 | create rabbitmq client failed | When users encounter this error code, it means that rabbitmq has some problems, please check it whether is work | +| RABBITMQ-03 | close connection failed | When users encounter this error code, it means that rabbitmq has some problems, please check it whether is work | +| RABBITMQ-04 | send messages failed | When users encounter this error code, it means that rabbitmq has some problems, please check it whether is work | +| RABBITMQ-05 | messages could not be acknowledged during checkpoint creation | When users encounter this error code, it means that job has some problems, please check it whether is work well | +| RABBITMQ-06 | messages could not be acknowledged with basicReject | When users encounter this error code, it means that job has some problems, please check it whether is work well | +| RABBITMQ-07 | parse uri failed | When users encounter this error code, it means that rabbitmq connect uri incorrect, please check it | +| RABBITMQ-08 | initialize ssl context failed | When users encounter this error code, it means that rabbitmq has some problems, please check it whether is work | +| RABBITMQ-09 | setup ssl factory failed | When users encounter this error code, it means that rabbitmq has some problems, please check it whether is work | diff --git a/seatunnel-connectors-v2/connector-rabbitmq/src/main/java/org/apache/seatunnel/connectors/seatunnel/rabbitmq/client/RabbitmqClient.java b/seatunnel-connectors-v2/connector-rabbitmq/src/main/java/org/apache/seatunnel/connectors/seatunnel/rabbitmq/client/RabbitmqClient.java index e7482eb1964..fbb482ffdb0 100644 --- a/seatunnel-connectors-v2/connector-rabbitmq/src/main/java/org/apache/seatunnel/connectors/seatunnel/rabbitmq/client/RabbitmqClient.java +++ b/seatunnel-connectors-v2/connector-rabbitmq/src/main/java/org/apache/seatunnel/connectors/seatunnel/rabbitmq/client/RabbitmqClient.java @@ -17,10 +17,6 @@ package org.apache.seatunnel.connectors.seatunnel.rabbitmq.client; -import static org.apache.seatunnel.connectors.seatunnel.rabbitmq.exception.RabbitmqConnectorErrorCode.CLOSE_CONNECTION_FAILED; -import static org.apache.seatunnel.connectors.seatunnel.rabbitmq.exception.RabbitmqConnectorErrorCode.CREATE_RABBITMQ_CLIENT_FAILED; -import static org.apache.seatunnel.connectors.seatunnel.rabbitmq.exception.RabbitmqConnectorErrorCode.SEND_MESSAGE_FAILED; - import org.apache.seatunnel.common.Handover; import org.apache.seatunnel.connectors.seatunnel.rabbitmq.config.RabbitmqConfig; import org.apache.seatunnel.connectors.seatunnel.rabbitmq.exception.RabbitmqConnectorException; @@ -40,13 +36,15 @@ import java.security.NoSuchAlgorithmException; import java.util.concurrent.TimeoutException; +import static org.apache.seatunnel.connectors.seatunnel.rabbitmq.exception.RabbitmqConnectorErrorCode.*; + @Slf4j @AllArgsConstructor public class RabbitmqClient { - private RabbitmqConfig config; - private ConnectionFactory connectionFactory; - private Connection connection; - private Channel channel; + private final RabbitmqConfig config; + private final ConnectionFactory connectionFactory; + private final Connection connection; + private final Channel channel; public RabbitmqClient(RabbitmqConfig config) { this.config = config; @@ -74,23 +72,19 @@ public DefaultConsumer getQueueingConsumer(Handover handover) { return consumer; } - public ConnectionFactory getConnectionFactory() - throws URISyntaxException, NoSuchAlgorithmException, KeyManagementException { + public ConnectionFactory getConnectionFactory() { ConnectionFactory factory = new ConnectionFactory(); if (!StringUtils.isEmpty(config.getUri())) { try { factory.setUri(config.getUri()); } catch (URISyntaxException e) { - log.error("Failed to parse uri", e); - throw e; + throw new RabbitmqConnectorException(PARSE_URI_FAILED, e); } catch (KeyManagementException e) { // this should never happen - log.error("Failed to initialize ssl context.", e); - throw e; + throw new RabbitmqConnectorException(INIT_SSL_CONTEXT_FAILED, e); } catch (NoSuchAlgorithmException e) { // this should never happen - log.error("Failed to setup ssl factory.", e); - throw e; + throw new RabbitmqConnectorException(SETUP_SSL_FACTORY_FAILED, e); } } else { factory.setHost(config.getHost()); diff --git a/seatunnel-connectors-v2/connector-rabbitmq/src/main/java/org/apache/seatunnel/connectors/seatunnel/rabbitmq/exception/RabbitmqConnectorErrorCode.java b/seatunnel-connectors-v2/connector-rabbitmq/src/main/java/org/apache/seatunnel/connectors/seatunnel/rabbitmq/exception/RabbitmqConnectorErrorCode.java index 2518099a2e6..e1abf85ef14 100644 --- a/seatunnel-connectors-v2/connector-rabbitmq/src/main/java/org/apache/seatunnel/connectors/seatunnel/rabbitmq/exception/RabbitmqConnectorErrorCode.java +++ b/seatunnel-connectors-v2/connector-rabbitmq/src/main/java/org/apache/seatunnel/connectors/seatunnel/rabbitmq/exception/RabbitmqConnectorErrorCode.java @@ -25,7 +25,10 @@ public enum RabbitmqConnectorErrorCode implements SeaTunnelErrorCode { CLOSE_CONNECTION_FAILED("RABBITMQ-03", "close connection failed"), SEND_MESSAGE_FAILED("RABBITMQ-04", "send messages failed"), MESSAGE_ACK_FAILED("RABBITMQ-05", "messages could not be acknowledged during checkpoint creation"), - MESSAGE_ACK_REJECTED("RABBITMQ-06", "messages could not be acknowledged with basicReject"); + MESSAGE_ACK_REJECTED("RABBITMQ-06", "messages could not be acknowledged with basicReject"), + PARSE_URI_FAILED("RABBITMQ-07", "parse uri failed"), + INIT_SSL_CONTEXT_FAILED("RABBITMQ-08", "initialize ssl context failed"), + SETUP_SSL_FACTORY_FAILED("RABBITMQ-09", "setup ssl factory failed"); private final String code; private final String description; From 43bd2d6b87dc72887e0cc9db6ab5f88da49c7d68 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=AF=95=E5=8D=9A?= Date: Wed, 23 Nov 2022 19:25:42 +0800 Subject: [PATCH 14/15] add RabbitConnectorException and fix doc style --- docs/en/connector-v2/sink/Rabbitmq.md | 31 +++++++++++-------- docs/en/connector-v2/source/Rabbitmq.md | 6 +++- .../seatunnel/rabbitmq/sink/RabbitmqSink.java | 6 +++- .../rabbitmq/source/RabbitmqSource.java | 10 ++++-- 4 files changed, 36 insertions(+), 17 deletions(-) diff --git a/docs/en/connector-v2/sink/Rabbitmq.md b/docs/en/connector-v2/sink/Rabbitmq.md index 2e62e16bda5..5a6c9a7defe 100644 --- a/docs/en/connector-v2/sink/Rabbitmq.md +++ b/docs/en/connector-v2/sink/Rabbitmq.md @@ -13,19 +13,20 @@ Used to write data to Rabbitmq. ## Options -| name | type | required | default value | -|-----------------------------|---------|----------|---------------| -| host | string | yes | - | -| port | int | yes | - | -| virtual_host | string | yes | - | -| username | string | yes | - | -| password | string | yes | - | -| queue_name | string | yes | - | -| url | string | no | - | -| network_recovery_interval | int | no | - | -| topology_recovery_enabled | boolean | no | - | -| automatic_recovery_enabled | boolean | no | - | -| connection_timeout | int | no | - | +| name | type | required | default value | +|-----------------------------|---------|-----------|---------------| +| host | string | yes | - | +| port | int | yes | - | +| virtual_host | string | yes | - | +| username | string | yes | - | +| password | string | yes | - | +| queue_name | string | yes | - | +| url | string | no | - | +| network_recovery_interval | int | no | - | +| topology_recovery_enabled | boolean | no | - | +| automatic_recovery_enabled | boolean | no | - | +| connection_timeout | int | no | - | +| common-options | | no | - | ### host [string] @@ -77,6 +78,10 @@ if true, enables connection recovery connection TCP establishment timeout in milliseconds; zero for infinite +### common options + +Sink plugin common parameters, please refer to [Sink Common Options](common-options.md) for details + ## Example simple: diff --git a/docs/en/connector-v2/source/Rabbitmq.md b/docs/en/connector-v2/source/Rabbitmq.md index d1cfb6f517c..47030cd46a1 100644 --- a/docs/en/connector-v2/source/Rabbitmq.md +++ b/docs/en/connector-v2/source/Rabbitmq.md @@ -41,7 +41,7 @@ The source must be non-parallel (parallelism set to 1) in order to achieve exact | requested_heartbeat | int | no | - | | prefetch_count | int | no | - | | delivery_timeout | long | no | - | - +| common-options | | no | - | ### host [string] @@ -123,6 +123,10 @@ prefetchCount the max number of messages to receive without acknowledgement deliveryTimeout maximum wait time, in milliseconds, for the next message delivery +### common options + +Source plugin common parameters, please refer to [Source Common Options](common-options.md) for details + ## Example simple: diff --git a/seatunnel-connectors-v2/connector-rabbitmq/src/main/java/org/apache/seatunnel/connectors/seatunnel/rabbitmq/sink/RabbitmqSink.java b/seatunnel-connectors-v2/connector-rabbitmq/src/main/java/org/apache/seatunnel/connectors/seatunnel/rabbitmq/sink/RabbitmqSink.java index 5244e62bdc7..971397da38d 100644 --- a/seatunnel-connectors-v2/connector-rabbitmq/src/main/java/org/apache/seatunnel/connectors/seatunnel/rabbitmq/sink/RabbitmqSink.java +++ b/seatunnel-connectors-v2/connector-rabbitmq/src/main/java/org/apache/seatunnel/connectors/seatunnel/rabbitmq/sink/RabbitmqSink.java @@ -25,6 +25,7 @@ import static org.apache.seatunnel.connectors.seatunnel.rabbitmq.config.RabbitmqConfig.VIRTUAL_HOST; import org.apache.seatunnel.api.common.PrepareFailException; +import org.apache.seatunnel.api.common.SeaTunnelAPIErrorCode; import org.apache.seatunnel.api.sink.SeaTunnelSink; import org.apache.seatunnel.api.sink.SinkWriter; import org.apache.seatunnel.api.table.type.SeaTunnelDataType; @@ -36,6 +37,7 @@ import org.apache.seatunnel.connectors.seatunnel.common.sink.AbstractSimpleSink; import org.apache.seatunnel.connectors.seatunnel.common.sink.AbstractSinkWriter; import org.apache.seatunnel.connectors.seatunnel.rabbitmq.config.RabbitmqConfig; +import org.apache.seatunnel.connectors.seatunnel.rabbitmq.exception.RabbitmqConnectorException; import org.apache.seatunnel.shade.com.typesafe.config.Config; @@ -60,7 +62,9 @@ public void prepare(Config pluginConfig) throws PrepareFailException { CheckResult result = CheckConfigUtil.checkAllExists(pluginConfig, HOST.key(), PORT.key(), VIRTUAL_HOST.key(), USERNAME.key(), PASSWORD.key(), QUEUE_NAME.key()); if (!result.isSuccess()) { - throw new PrepareFailException(getPluginName(), PluginType.SINK, result.getMsg()); + throw new RabbitmqConnectorException(SeaTunnelAPIErrorCode.CONFIG_VALIDATION_FAILED, + String.format("PluginName: %s, PluginType: %s, Message: %s", + getPluginName(), PluginType.SINK, result.getMsg())); } rabbitMQConfig = new RabbitmqConfig(pluginConfig); } diff --git a/seatunnel-connectors-v2/connector-rabbitmq/src/main/java/org/apache/seatunnel/connectors/seatunnel/rabbitmq/source/RabbitmqSource.java b/seatunnel-connectors-v2/connector-rabbitmq/src/main/java/org/apache/seatunnel/connectors/seatunnel/rabbitmq/source/RabbitmqSource.java index 4883d6a294b..28a5ddc7e0e 100644 --- a/seatunnel-connectors-v2/connector-rabbitmq/src/main/java/org/apache/seatunnel/connectors/seatunnel/rabbitmq/source/RabbitmqSource.java +++ b/seatunnel-connectors-v2/connector-rabbitmq/src/main/java/org/apache/seatunnel/connectors/seatunnel/rabbitmq/source/RabbitmqSource.java @@ -27,6 +27,7 @@ import org.apache.seatunnel.api.common.JobContext; import org.apache.seatunnel.api.common.PrepareFailException; +import org.apache.seatunnel.api.common.SeaTunnelAPIErrorCode; import org.apache.seatunnel.api.serialization.DeserializationSchema; import org.apache.seatunnel.api.source.Boundedness; import org.apache.seatunnel.api.source.SeaTunnelSource; @@ -41,6 +42,7 @@ import org.apache.seatunnel.common.constants.PluginType; import org.apache.seatunnel.connectors.seatunnel.common.schema.SeaTunnelSchema; import org.apache.seatunnel.connectors.seatunnel.rabbitmq.config.RabbitmqConfig; +import org.apache.seatunnel.connectors.seatunnel.rabbitmq.exception.RabbitmqConnectorException; import org.apache.seatunnel.connectors.seatunnel.rabbitmq.split.RabbitmqSplit; import org.apache.seatunnel.connectors.seatunnel.rabbitmq.split.RabbitmqSplitEnumeratorState; import org.apache.seatunnel.format.json.JsonDeserializationSchema; @@ -59,7 +61,9 @@ public class RabbitmqSource implements SeaTunnelSource Date: Thu, 24 Nov 2022 11:01:59 +0800 Subject: [PATCH 15/15] fix dist pom conflict --- seatunnel-dist/pom.xml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/seatunnel-dist/pom.xml b/seatunnel-dist/pom.xml index 4a2c7c02887..b37a99356f0 100644 --- a/seatunnel-dist/pom.xml +++ b/seatunnel-dist/pom.xml @@ -365,11 +365,16 @@ org.apache.seatunnel - connector-rabbitmq connector-http-jira ${project.version} provided + + org.apache.seatunnel + connector-rabbitmq + ${project.version} + provided +