queue

airbyte-commons/src/main/java/io/airbyte/commons/lang/CloseableQueue.java

@@ -0,0 +1,31 @@
+/*
+ * MIT License
+ *
+ * Copyright (c) 2020 Airbyte
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+package io.airbyte.commons.lang;
+
+import java.util.Queue;
+
+public interface CloseableQueue<E> extends Queue<E>, AutoCloseable {
+
+}

airbyte-queue/build.gradle

@@ -0,0 +1,7 @@
+plugins {
+    id 'java-library'
+}
+
+dependencies {
+    implementation 'com.baidu:leansoft-bigqueue:0.7.3'
+}

airbyte-queue/src/main/java/io/airbyte/queue/BigQueue.java

@@ -0,0 +1,114 @@
+/*
+ * MIT License
+ *
+ * Copyright (c) 2020 Airbyte
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+package io.airbyte.queue;
+
+import com.google.common.base.Preconditions;
+import com.google.common.collect.AbstractIterator;
+import com.leansoft.bigqueue.BigQueueImpl;
+import com.leansoft.bigqueue.IBigQueue;
+import io.airbyte.commons.lang.CloseableQueue;
+import java.io.IOException;
+import java.nio.file.Path;
+import java.util.AbstractQueue;
+import java.util.Iterator;
+import java.util.concurrent.atomic.AtomicBoolean;
+
+/**
+ * Wraps BigQueueImpl behind Airbyte persistent queue interface. BigQueueImpl is threadsafe.
+ */
+public class BigQueue extends AbstractQueue<byte[]> implements CloseableQueue<byte[]> {
+
+  private final IBigQueue queue;
+  private final AtomicBoolean closed = new AtomicBoolean(false);
+
+  public BigQueue(Path persistencePath, String queueName) throws IOException {
+    queue = new BigQueueImpl(persistencePath.toString(), queueName);
+  }
+
+  @Override
+  public boolean offer(byte[] bytes) {
+    Preconditions.checkState(!closed.get());
+    try {
+      queue.enqueue(bytes);
+    } catch (IOException e) {
+      throw new RuntimeException(e);
+    }
+    return true;
+  }
+
+  @Override
+  public byte[] poll() {
+    Preconditions.checkState(!closed.get());
+    try {
+      return queue.dequeue();
+    } catch (IOException e) {
+      throw new RuntimeException(e);
+    }
+  }
+
+  @Override
+  public byte[] peek() {
+    Preconditions.checkState(!closed.get());
+    try {
+      return queue.peek();
+    } catch (IOException e) {
+      throw new RuntimeException(e);
+    }
+  }
+
+  @Override
+  public int size() {
+    Preconditions.checkState(!closed.get());
+    return Math.toIntExact(queue.size());
+  }
+
+  @Override
+  public Iterator<byte[]> iterator() {
+    Preconditions.checkState(!closed.get());
+
+    return new AbstractIterator<>() {
+
+      @Override
+      protected byte[] computeNext() {
+        final byte[] poll = poll();
+        if (poll == null) {
+          return endOfData();
+        }
+        return poll;
+      }
+
+    };
+  }
+
+  @Override
+  public void close() throws Exception {
+    closed.set(true);
+    // todo (cgardens) - this barfs out a huge warning. known issue with the lib:
+    // https://github.com/bulldog2011/bigqueue/issues/35.
+    queue.close();
+    queue.gc();
+  }
+
+}

airbyte-queue/src/test/java/io/airbyte/queue/BigQueueTest.java

@@ -0,0 +1,91 @@
+/*
+ * MIT License
+ *
+ * Copyright (c) 2020 Airbyte
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in all
+ * copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+package io.airbyte.queue;
+
+import static org.junit.jupiter.api.Assertions.assertDoesNotThrow;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+
+import com.google.common.base.Charsets;
+import io.airbyte.commons.lang.CloseableQueue;
+import java.io.IOException;
+import java.nio.file.Files;
+import java.util.Objects;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+
+class BigQueueTest {
+
+  private CloseableQueue<byte[]> queue;
+
+  @BeforeEach
+  void setup() throws IOException {
+    queue = new BigQueue(Files.createTempDirectory("test"), "test");
+  }
+
+  @AfterEach
+  void teardown() throws Exception {
+    queue.close();
+  }
+
+  @Test
+  void testPoll() {
+    queue.offer(toBytes("hello"));
+    assertEquals("hello", new String(Objects.requireNonNull(queue.poll()), Charsets.UTF_8));
+  }
+
+  @Test
+  void testPeek() {
+    queue.offer(toBytes("hello"));
+    assertEquals("hello", new String(Objects.requireNonNull(queue.peek()), Charsets.UTF_8));
+    assertEquals("hello", new String(Objects.requireNonNull(queue.peek()), Charsets.UTF_8));
+    assertEquals("hello", new String(Objects.requireNonNull(queue.poll()), Charsets.UTF_8));
+  }
+
+  @Test
+  void testSize() {
+    assertEquals(0, queue.size());
+    queue.offer(toBytes("hello"));
+    assertEquals(1, queue.size());
+    queue.offer(toBytes("hello"));
+    assertEquals(2, queue.size());
+  }
+
+  @Test
+  void testClosed() throws Exception {
+    queue.close();
+    assertDoesNotThrow(() -> queue.close());
+    assertThrows(IllegalStateException.class, () -> queue.offer(toBytes("hello")));
+    assertThrows(IllegalStateException.class, () -> queue.poll());
+    assertThrows(IllegalStateException.class, () -> queue.iterator());
+  }
+
+  @SuppressWarnings("SameParameterValue")
+  private static byte[] toBytes(String string) {
+    return string.getBytes(Charsets.UTF_8);
+  }
+
+}

settings.gradle

@@ -11,6 +11,7 @@ include ':airbyte-config:persistence'
 include ':airbyte-db'
 include ':airbyte-integrations'
 include ':airbyte-protocol:models'
+include ':airbyte-queue'
 include ':airbyte-scheduler'
 include ':airbyte-server'
 include ':airbyte-singer'