ci: fix testing

.github/workflows/ci.yml

@@ -8,5 +8,10 @@ jobs:
     steps:
       - name: Checkout
         uses: actions/checkout@v2
+      - id: changed-files
+        uses: jitterbit/get-changed-files@v1
+        continue-on-error: true
       - id: tests
-        run: ./scripts/tests.sh
+        run: ./scripts/tests.sh
+        env:
+          CHANGED_FILES: ${{ steps.changed-files.outputs.all }}

jinahub/indexers/dbms/PostgreSQLDBMSIndexer/README.md

@@ -0,0 +1,11 @@
+# PostgreSQLDBMSIndexer
+
+Indexer wrapper around the PostgreSQL DBMS. Postgres is an open source object-relational database. You can read more about it here: https://www.postgresql.org/
+
+## NOTES
+
+This indexer assumes a PRIMARY KEY on the `id` field, thus you cannot add two `Document` of the same id. Make sure you clean up any existing data if you want to start fresh. 
+
+## USAGE
+
+TBD

jinahub/indexers/dbms/PostgreSQLIndexer/__init__.py → jinahub/indexers/dbms/PostgreSQLDBMSIndexer/__init__.py

@@ -35,16 +35,14 @@ def __init__(
         *args,
         **kwargs,
     ):
-        from .postgreshandler import PostgreSQLDBMSHandler
-
         super().__init__(*args, **kwargs)
         self.hostname = hostname
         self.port = port
         self.username = username
         self.password = password
         self.database = database
         self.table = table
-        self.logger = JinaLogger('PostgreSQLDBMSIndexer')
+        self.logger = JinaLogger(self.metas.name)
         self.handler = PostgreSQLDBMSHandler(
             hostname=self.hostname,
             port=self.port,
@@ -57,7 +55,7 @@ def __init__(
     def _get_generator(self) -> Generator[Tuple[str, np.array, bytes], None, None]:
         with self.handler as handler:
             # always order the dump by id as integer
-            handler.cursor.execute(f"SELECT * from {handler.table} ORDER BY ID::int")
+            handler.cursor.execute(f'SELECT * from {handler.table} ORDER BY ID::int')
             records = handler.cursor.fetchall()
             for rec in records:
                 yield rec[0], np.frombuffer(bytes(rec[1])), bytes(rec[2])
@@ -70,23 +68,11 @@ def size(self):
         """
         with self.handler as postgres_handler:
             postgres_handler.cursor.execute(
-                f"SELECT COUNT(*) from {self.handler.table}"
+                f'SELECT COUNT(*) from {self.handler.table}'
             )
             records = postgres_handler.cursor.fetchall()
             return records[0][0]
 
-    def get_handler(self) -> 'PostgreSQLDBMSHandler':
-        """Get the handler to PostgreSQLDBMS."""
-        return self.handler
-
-    def get_add_handler(self) -> 'PostgreSQLDBMSHandler':
-        """Get the handler to PostgresSQLDBMS."""
-        return self.handler
-
-    def get_create_handler(self) -> 'PostgreSQLDBMSHandler':
-        """Get the handler to PostgresSQLDBMS."""
-        return self.handler
-
     @requests(on='/index')
     def add(self, docs: DocumentArray, **kwargs):
         """Add a Document to PostgreSQLDBMS.

jinahub/indexers/dbms/PostgreSQLIndexer/postgreshandler.py → jinahub/indexers/dbms/PostgreSQLDBMSIndexer/postgreshandler.py

@@ -57,6 +57,9 @@ def connect(self) -> 'PostgreSQLDBMSHandler':
         from psycopg2 import Error
 
         try:
+            # by default psycopg2 is not auto-committing
+            # this means we can have rollbacks
+            # and maintain ACID-ity
             self.connection = psycopg2.connect(
                 user=self.username,
                 password=self.password,
@@ -89,10 +92,10 @@ def use_table(self):
         else:
             try:
                 self.cursor.execute(
-                    f"CREATE TABLE {self.table} ( \
+                    f'CREATE TABLE {self.table} ( \
                     ID VARCHAR PRIMARY KEY,  \
                     VECS BYTEA,  \
-                    METAS BYTEA);"
+                    METAS BYTEA);'
                 )
                 self.logger.info('Successfully created table')
             except (Exception, Error) as error:
@@ -177,7 +180,7 @@ def query(self, docs: DocumentArray, **kwargs):
         for doc in docs:
             # retrieve metadata
             self.cursor.execute(
-                f'SELECT METAS FROM {self.table} WHERE ID = %s;', doc.id
+                f'SELECT METAS FROM {self.table} WHERE ID = %s;', (doc.id,)
             )
             result = self.cursor.fetchone()
             data = bytes(result[0])

jinahub/indexers/dbms/PostgreSQLIndexer/tests/test_PostgreSQLdbindexer.py → jinahub/indexers/dbms/PostgreSQLDBMSIndexer/tests/test_PostgreSQLdbindexer.py

@@ -1,3 +1,5 @@
+from copy import copy, deepcopy
+
 import os
 
 import numpy as np
@@ -62,6 +64,8 @@ def test_postgress(tmpdir):
 
     original_docs = list(get_documents(chunks=0, same_content=False))
 
+    postgres_indexer.handler.delete(original_docs)
+
     added = postgres_indexer.handler.add(original_docs)
     assert added == 10
 

jinahub/indexers/query/compound/NumpyPostgresQueryIndexer/requirements.txt

@@ -1 +1,2 @@
-jina==2.0.0rc5
+jina==2.0.0rc5.dev54
+git+git://github.com/jina-ai/jina-commons

jinahub/indexers/query/keyvalue/FileQueryIndexer/Dockerfile

@@ -1,12 +1,14 @@
-FROM jinaai/jina:2.0.0
+FROM jinaai/jina:1.0.2
 
 # install the third-party requirements
-RUN apt-get update && apt-get install --no-install-recommends -y gcc build-essential
+RUN apt-get update && apt-get install --no-install-recommends -y gcc build-essential git
 
 # setup the workspace
 COPY . /workspace
 WORKDIR /workspace
 
+RUN pip install -r requirements.txt
+
 # for testing the image
 RUN pip install pytest && pytest tests