REST API tests for IAM (#4090)

.github/workflows/main.yml

@@ -77,8 +77,6 @@ jobs:
         run: |
           docker-compose -f docker-compose.yml -f docker-compose.dev.yml -f components/serverless/docker-compose.serverless.yml up -d
           /bin/bash -c 'while [[ "$(curl -s -o /dev/null -w ''%{http_code}'' ${API_ABOUT_PAGE})" != "401" ]]; do sleep 5; done'
-          cat tests/rest_api/assets/cvat_db.sql | docker exec -i cvat_db psql -q -U root -d cvat
-          cat tests/rest_api/assets/cvat_data.tar.bz2 | docker run --rm -i --volumes-from cvat ubuntu tar -xj --strip 3 -C /home/django/data
           pip3 install --user -r tests/rest_api/requirements.txt
           pytest tests/rest_api/
           docker-compose -f docker-compose.yml -f docker-compose.dev.yml -f components/serverless/docker-compose.serverless.yml down -v

tests/rest_api/README.md

@@ -35,7 +35,7 @@ available in the system like comments, users, issues, please use the following
 procedure to add them:
 
 1. Run a clean CVAT instance
-1. Restore DB and data volume using commands below
+1. Restore DB and data volume using commands below or running tests
 1. Add new objects (e.g. issues, comments, tasks, projects)
 1. Backup DB and data volume using commands below
 1. Don't forget to dump new objects into corresponding json files inside
@@ -58,35 +58,31 @@ for i, color in enumerate(colormap):
     img.save(f'{i}.png')
 ```
 
+## How to backup DB and data volume?
+
 To backup DB and data volume, please use commands below.
 
 ```console
-docker exec -i cvat_db pg_dump -c -U root -d cvat > assets/cvat_db.sql
+docker exec cvat_db pg_dump -c -C -Fc -U root -d cvat > assets/cvat_db.dump
 docker run --rm --volumes-from cvat ubuntu tar -cjv /home/django/data > assets/cvat_data.tar.bz2
 ```
 
-To restore DB and data volume, please use commands below.
+# How to update *.json files in the assets directory?
 
-```console
-cat assets/cvat_db.sql | docker exec -i cvat_db psql -q -U root -d cvat
-cat assets/cvat_data.tar.bz2 | docker run --rm -i --volumes-from cvat ubuntu tar -xj --strip 3 -C /home/django/data
+If you have updated the test database and want to update the assets/*.json
+files as well, run the appropriate script:
+
+```
+python utils/dump_objects.py
 ```
 
-To dump an object into JSON, please look at the sample code below. You also
-can find similar code in `utils/dump_objects.py` script. All users in the
-testing system has the same password `!Q@W#E$R`.
+# How to restore DB and data volume?
 
-```python
-import requests
-import json
-
-with requests.Session() as session:
-    session.auth = ('admin1', '!Q@W#E$R')
+To restore DB and data volume, please use commands below.
 
-    for obj in ['user', 'project', 'task', 'job']:
-        response = session.get(f'http://localhost:8080/api/v1/{obj}s')
-        with open(f'{obj}s.json', 'w') as f:
-            json.dump(response.json(), f, indent=2, sort_keys=True)
+```console
+cat assets/cvat_db/cvat_db.dump | docker exec -i cvat_db pg_restore -1 -c -U root -d cvat
+cat assets/cvat_data.tar.bz2 | docker run --rm -i --volumes-from cvat ubuntu tar -xj --strip 3 -C /home/django/data
 ```
 
 ## FAQ
@@ -97,7 +93,7 @@ with requests.Session() as session:
    you have json description of all objects together with cvat_db.sql, it will
    be possible to recreate them manually.
 
-1. How to upgrade cvat_data.tar.bz2 and cvat_db.sql?
+1. How to upgrade cvat_data.tar.bz2 and cvat_db.dump?
 
    After every commit which changes the layout of DB and data directory it is
    possible to break these files. But failed tests should be a clear indicator
@@ -109,3 +105,9 @@ with requests.Session() as session:
    Construction of some objects can be complex and takes time (backup
    and restore should be much faster). Construction of objects in UI is more
    intuitive.
+
+1. How we solve the problem of dependent tests?
+
+   Since some tests change the database, these tests may be dependent on each
+   other, so in current implementation we avoid such problem by restoring
+   the database after each test function (see `conftest.py`)

tests/rest_api/assets/cvat_db/cvat_db.sql

@@ -0,0 +1,7 @@
+SELECT pg_terminate_backend(pg_stat_activity.pid)
+FROM pg_stat_activity
+WHERE pg_stat_activity.datname = 'cvat' AND pid <> pg_backend_pid();
+
+DROP DATABASE cvat;
+
+CREATE DATABASE cvat WITH TEMPLATE test_db;

tests/rest_api/assets/invitations.json

@@ -1,8 +1,48 @@
 {
-  "count": 7,
+  "count": 9,
   "next": null,
   "previous": null,
   "results": [
+    {
+      "created_date": "2022-01-19T13:54:42.015131Z",
+      "key": "BrwoDmMNQQ1v9WXOukp9DwQVuqB3RDPjpUECCEq6QcAuG0Pi8k1IYtQ9uz9jg0Bv",
+      "organization": 2,
+      "owner": {
+        "first_name": "Business",
+        "id": 10,
+        "last_name": "First",
+        "url": "http://localhost:8080/api/v1/users/10",
+        "username": "business1"
+      },
+      "role": "maintainer",
+      "user": {
+        "first_name": "User",
+        "id": 5,
+        "last_name": "Fourth",
+        "url": "http://localhost:8080/api/v1/users/5",
+        "username": "user4"
+      }
+    },
+    {
+      "created_date": "2022-01-19T13:54:42.005381Z",
+      "key": "5FjIXya6fTGvlRpauFvi2QN1wDOqo1V9REB5rJinDR8FZO9gr0qmtWpghsCte8Y1",
+      "organization": 2,
+      "owner": {
+        "first_name": "Business",
+        "id": 10,
+        "last_name": "First",
+        "url": "http://localhost:8080/api/v1/users/10",
+        "username": "business1"
+      },
+      "role": "supervisor",
+      "user": {
+        "first_name": "User",
+        "id": 4,
+        "last_name": "Third",
+        "url": "http://localhost:8080/api/v1/users/4",
+        "username": "user3"
+      }
+    },
     {
       "created_date": "2021-12-14T19:55:13.745912Z",
       "key": "h43G28di7vfs4Jv5VrKZ26xvGAfm6Yc2FFv14z9EKhiuIEDQ22pEnzmSCab8MnK1",

tests/rest_api/assets/memberships.json

@@ -1,8 +1,38 @@
 {
-  "count": 9,
+  "count": 11,
   "next": null,
   "previous": null,
   "results": [
+    {
+      "id": 11,
+      "invitation": "BrwoDmMNQQ1v9WXOukp9DwQVuqB3RDPjpUECCEq6QcAuG0Pi8k1IYtQ9uz9jg0Bv",
+      "is_active": true,
+      "joined_date": "2022-01-19T13:54:42.015131Z",
+      "organization": 2,
+      "role": "maintainer",
+      "user": {
+        "first_name": "User",
+        "id": 5,
+        "last_name": "Fourth",
+        "url": "http://localhost:8080/api/v1/users/5",
+        "username": "user4"
+      }
+    },
+    {
+      "id": 10,
+      "invitation": "5FjIXya6fTGvlRpauFvi2QN1wDOqo1V9REB5rJinDR8FZO9gr0qmtWpghsCte8Y1",
+      "is_active": true,
+      "joined_date": "2022-01-19T13:54:42.005381Z",
+      "organization": 2,
+      "role": "supervisor",
+      "user": {
+        "first_name": "User",
+        "id": 4,
+        "last_name": "Third",
+        "url": "http://localhost:8080/api/v1/users/4",
+        "username": "user3"
+      }
+    },
     {
       "id": 9,
       "invitation": "h43G28di7vfs4Jv5VrKZ26xvGAfm6Yc2FFv14z9EKhiuIEDQ22pEnzmSCab8MnK1",

tests/rest_api/assets/users.json

@@ -14,7 +14,7 @@
       "is_active": true,
       "is_staff": true,
       "is_superuser": true,
-      "last_login": "2021-12-22T07:32:58.602211Z",
+      "last_login": "2021-12-22T08:11:58.502575Z",
       "last_name": "First",
       "url": "http://localhost:8080/api/v1/users/1",
       "username": "admin1"
@@ -158,7 +158,7 @@
       "is_active": true,
       "is_staff": false,
       "is_superuser": false,
-      "last_login": "2021-12-14T19:44:48.526708Z",
+      "last_login": "2022-01-19T13:52:59.477881Z",
       "last_name": "First",
       "url": "http://localhost:8080/api/v1/users/10",
       "username": "business1"

tests/rest_api/conftest.py

@@ -0,0 +1,119 @@
+# Copyright (C) 2021 Intel Corporation
+#
+# SPDX-License-Identifier: MIT
+
+from subprocess import run, CalledProcessError
+import pytest
+import json
+import os.path as osp
+from .utils.config import ASSETS_DIR
+
+def cvat_db_container(command):
+    run(('docker exec cvat_db ' + command).split(), check=True) #nosec
+
+def docker_cp(source, target):
+    run(' '.join(['docker container cp', source, target]).split(), check=True) #nosec
+
+def restore_data_volume():
+    command = 'docker run --rm --volumes-from cvat --mount ' \
+        f'type=bind,source={ASSETS_DIR},target=/mnt/ ubuntu tar ' \
+        '--strip 3 -C /home/django/data -xjf /mnt/cvat_data.tar.bz2'
+    run(command.split(), check=True) #nosec
+
+def drop_test_db():
+    cvat_db_container('pg_restore -c -U root -d cvat /cvat_db/cvat_db.dump')
+    cvat_db_container('rm -rf /cvat_db')
+    cvat_db_container('dropdb test_db')
+
+def create_test_db():
+    docker_cp(source=osp.join(ASSETS_DIR, 'cvat_db'), target='cvat_db:/')
+    cvat_db_container('createdb test_db')
+    cvat_db_container('pg_restore -U root -d test_db /cvat_db/cvat_db.dump')
+
+@pytest.fixture(scope='session', autouse=True)
+def init_test_db():
+    try:
+        restore_data_volume()
+        create_test_db()
+    except CalledProcessError:
+        drop_test_db()
+        pytest.exit(f"Cannot to initialize test DB")
+
+    yield
+
+    drop_test_db()
+
+@pytest.fixture(scope='function', autouse=True)
+def restore_cvat_db():
+    cvat_db_container('psql -U root -d postgres -f /cvat_db/cvat_db.sql')
+
+@pytest.fixture(scope='module')
+def users():
+    with open(osp.join(ASSETS_DIR, 'users.json')) as f:
+        return json.load(f)['results']
+
+@pytest.fixture(scope='module')
+def organizations():
+    with open(osp.join(ASSETS_DIR, 'organizations.json')) as f:
+        data = json.load(f)
+
+    def _organizations(org_id=None):
+        if org_id:
+            return [org for org in data if org['id'] == org_id][0]
+        return data
+
+    return _organizations
+
+@pytest.fixture(scope='module')
+def memberships():
+    with open(osp.join(ASSETS_DIR, 'memberships.json')) as f:
+        return json.load(f)['results']
+
+@pytest.fixture(scope='module')
+def users_by_name(users):
+    return {user['username']: user for user in users}
+
+@pytest.fixture(scope='module')
+def find_users(test_db):
+    def find(**kwargs):
+        assert len(kwargs) > 0
+        assert any(kwargs)
+
+        data = test_db
+        kwargs = dict(filter(lambda a: a[1] is not None, kwargs.items()))
+        for field, value in kwargs.items():
+            if field.startswith('exclude_'):
+                field = field.split('_', maxsplit=1)[1]
+                exclude_rows = set(v['id'] for v in
+                    filter(lambda a: a[field] == value, test_db))
+                data = list(filter(lambda a: a['id'] not in exclude_rows, data))
+            else:
+                data = list(filter(lambda a: a[field] == value, data))
+
+        return data
+    return find
+
+
+@pytest.fixture(scope='module')
+def test_db(users, users_by_name, memberships):
+    data = []
+    fields = ['username', 'id', 'privilege', 'role', 'org', 'membership_id']
+    def add_row(**kwargs):
+        data.append({field: kwargs.get(field) for field in fields})
+
+    for user in users:
+        for group in user['groups']:
+            add_row(username=user['username'], id=user['id'], privilege=group)
+
+    for membership in memberships:
+        username = membership['user']['username']
+        for group in users_by_name[username]['groups']:
+            add_row(username=username, role=membership['role'], privilege=group,
+                id=membership['user']['id'], org=membership['organization'],
+                membership_id=membership['id'])
+
+    return data
+
+
+
+

tests/rest_api/test_0000_check_objects_integrity.py

@@ -2,23 +2,20 @@
 #
 # SPDX-License-Identifier: MIT
 
-import os
+import os.path as osp
 import glob
-import requests
 import json
 from deepdiff import DeepDiff
 from .utils import config
+import pytest
 
-def test_check_objects_integrity():
-    with requests.Session() as session:
-        session.auth = ('admin1', config.USER_PASS)
+@pytest.mark.parametrize('path', glob.glob(osp.join(config.ASSETS_DIR, '*.json')))
+def test_check_objects_integrity(path):
+    with open(path) as f:
+        endpoint = osp.basename(path).rsplit('.')[0]
+        response = config.get_method('admin1', endpoint, page_size='all')
+        json_objs = json.load(f)
+        resp_objs = response.json()
 
-        for filename in glob.glob(os.path.join(config.ASSETS_DIR, '*.json')):
-            with open(filename) as f:
-                endpoint = os.path.basename(filename).rsplit('.')[0]
-                response = session.get(config.get_api_url(endpoint, page_size='all'))
-                json_objs = json.load(f)
-                resp_objs = response.json()
-
-                assert DeepDiff(json_objs, resp_objs, ignore_order=True,
-                    exclude_regex_paths="root\['results'\]\[\d+\]\['last_login'\]") == {}
+        assert DeepDiff(json_objs, resp_objs, ignore_order=True,
+            exclude_regex_paths="root\['results'\]\[\d+\]\['last_login'\]") == {}