Refactor paginator (#425)

* capsule some of the request calls

* final touches

* make mypy happy

* add missing file

* implement the new paginator design

* adjust tests to the new design

* trunk catches

* remove wrong file

* small improvements

* make mypy happy

* adjust expected result for doctest

* further smaller fixes

* fix vocab test

* automate cancelling outdated workflow runs

* fix doctest

* fix test expectations

* remove tests that don't really make sense. I don't think we should process user supplied URLs too much.

* fix trunk check

* make API and data schema smarter. Delay init of DBschema to raise API connection error at right place.

* full circle and we still don't have the correct search results

* reenable tests

* stupid mistakes

* remove unmerged paths

* add missing page increase

* add good changes

* less error prone

* raw is better for debugging

* fix import error

* remove obsolete test

.github/workflows/docs_check.yaml

@@ -15,6 +15,13 @@ on:
       - main
       - develop
       - "*"
+concurrency:
+  # github.workflow: name of the workflow
+  # github.event.pull_request.number || github.ref: pull request number or branch name if not a pull request
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+
+  # Cancel in-progress runs when a new workflow with the same group name is triggered
+  cancel-in-progress: true
 
 jobs:
   build:

.github/workflows/doctest.yaml

@@ -14,6 +14,14 @@ on:
       - main
       - develop
 
+concurrency:
+  # github.workflow: name of the workflow
+  # github.event.pull_request.number || github.ref: pull request number or branch name if not a pull request
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+
+  # Cancel in-progress runs when a new workflow with the same group name is triggered
+  cancel-in-progress: true
+
 jobs:
   doctest:
     strategy:

.github/workflows/mypy.yaml

@@ -14,6 +14,14 @@ on:
       - main
       - develop
 
+concurrency:
+  # github.workflow: name of the workflow
+  # github.event.pull_request.number || github.ref: pull request number or branch name if not a pull request
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+
+  # Cancel in-progress runs when a new workflow with the same group name is triggered
+  cancel-in-progress: true
+
 jobs:
   mypy-test:
     strategy:

.github/workflows/test_coverage.yaml

@@ -15,6 +15,14 @@ on:
       - main
       - develop
 
+concurrency:
+  # github.workflow: name of the workflow
+  # github.event.pull_request.number || github.ref: pull request number or branch name if not a pull request
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+
+  # Cancel in-progress runs when a new workflow with the same group name is triggered
+  cancel-in-progress: true
+
 jobs:
   test-coverage:
     runs-on: ubuntu-latest

.github/workflows/test_examples.yml

@@ -12,6 +12,14 @@ on:
       - main
       - develop
 
+concurrency:
+  # github.workflow: name of the workflow
+  # github.event.pull_request.number || github.ref: pull request number or branch name if not a pull request
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+
+  # Cancel in-progress runs when a new workflow with the same group name is triggered
+  cancel-in-progress: true
+
 jobs:
   test-examples:
     runs-on: ${{ matrix.os }}

.github/workflows/tests.yml

@@ -18,6 +18,14 @@ on:
       - develop
       - "*"
 
+concurrency:
+  # github.workflow: name of the workflow
+  # github.event.pull_request.number || github.ref: pull request number or branch name if not a pull request
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+
+  # Cancel in-progress runs when a new workflow with the same group name is triggered
+  cancel-in-progress: true
+
 jobs:
   install:
     runs-on: ${{ matrix.os }}

conftest.py

@@ -8,6 +8,7 @@
 
 The fixtures are all functional fixtures that stay consistent between all tests.
 """
+import logging
 import os
 
 import pytest
@@ -49,14 +50,15 @@ def cript_api():
     """
     storage_token = os.getenv("CRIPT_STORAGE_TOKEN")
 
-    with cript.API(host=None, api_token=None, storage_token=storage_token) as api:
+    with cript.API(host=None, api_token=None, storage_token=storage_token, default_log_level=logging.DEBUG) as api:
         # overriding AWS S3 cognito variables to be sure we do not upload test data to production storage
         # staging AWS S3 cognito storage variables
         api._IDENTITY_POOL_ID = "us-east-1:25043452-a922-43af-b8a6-7e938a9e55c1"
         api._COGNITO_LOGIN_PROVIDER = "cognito-idp.us-east-1.amazonaws.com/us-east-1_vyK1N9p22"
         api._BUCKET_NAME = "cript-stage-user-data"
         # using the tests folder name within our cloud storage
         api._BUCKET_DIRECTORY_NAME = "tests"
+        api.extra_api_log_debug_info = True
 
         yield api