MobileTeleSystems · maxim-lixakov · Nov 26, 2024 · dolfinus · Nov 27, 2024 · maxim-lixakov
diff --git a/.env.docker b/.env.docker
@@ -5,6 +5,15 @@ ENV=LOCAL
 SYNCMASTER__SERVER__DEBUG=true
 SYNCMASTER__SERVER__LOG_URL_TEMPLATE=https://grafana.example.com?correlation_id={{ correlation_id }}&run_id={{ run.id }}
 
+# Session
+SYNCMASTER__SERVER__SESSION__SECRET_KEY=session_secret_key
+
+# Postgres
+SYNCMASTER__DATABASE__URL=postgresql+asyncpg://syncmaster:changeme@db:5432/syncmaster
+
+# RabbitMQ
+SYNCMASTER__BROKER__URL=amqp://guest:guest@rabbitmq:5672/
+
 # Logging
 SYNCMASTER__LOGGING__SETUP=True
 SYNCMASTER__LOGGING__PRESET=colored
@@ -15,13 +24,6 @@ SYNCMASTER__ENCRYPTION__CRYPTO_KEY=UBgPTioFrtH2unlC4XFDiGf5sYfzbdSf_VgiUSaQc94=
 # Scheduler settings
 SYNCMASTER__SCHEDULER__TRANSFER_FETCHING_TIMEOUT_SECONDS=200
 
-# Session
-SYNCMASTER__SERVER__SESSION__SECRET_KEY=session_secret_key
-
-# Postgres
-SYNCMASTER__DATABASE__URL=postgresql+asyncpg://syncmaster:changeme@db:5432/syncmaster
-
-# TODO: add to KeycloakAuthProvider documentation about creating new realms, add users, etc.
 # KEYCLOAK Auth
 SYNCMASTER__AUTH__KEYCLOAK__SERVER_URL=http://keycloak:8080
 SYNCMASTER__AUTH__KEYCLOAK__REALM_NAME=manually_created
@@ -36,9 +38,6 @@ SYNCMASTER__AUTH__PROVIDER=syncmaster.backend.providers.auth.keycloak_provider.K
 SYNCMASTER__AUTH__PROVIDER=syncmaster.backend.providers.auth.dummy_provider.DummyAuthProvider
 SYNCMASTER__AUTH__ACCESS_TOKEN__SECRET_KEY=secret
 
-# RabbitMQ
-SYNCMASTER__BROKER__URL=amqp://guest:guest@rabbitmq:5672/
-
 # S3
 MINIO_ACCESS_KEY=syncmaster
 MINIO_SECRET_KEY=changeme

diff --git a/.env.local b/.env.local
@@ -5,21 +5,24 @@ export ENV=LOCAL
 export SYNCMASTER__SERVER__DEBUG=true
 export SYNCMASTER__SERVER__LOG_URL_TEMPLATE="https://grafana.example.com?correlation_id={{ correlation_id }}&run_id={{ run.id }}"
 
+# Session
+export SYNCMASTER__SERVER__SESSION__SECRET_KEY=session_secret_key
+
+# Postgres
+export SYNCMASTER__DATABASE__URL=postgresql+asyncpg://syncmaster:changeme@localhost:5432/syncmaster
+
+# RabbitMQ
+export SYNCMASTER__BROKER__URL=amqp://guest:guest@localhost:5672/
+
 # Logging
 export SYNCMASTER__LOGGING__SETUP=True
 export SYNCMASTER__LOGGING__PRESET=colored
 
-# Scheduler settings
-export SYNCMASTER__SCHEDULER__TRANSFER_FETCHING_TIMEOUT_SECONDS=200
-
-# Session
-export SYNCMASTER__SERVER__SESSION__SECRET_KEY=session_secret_key
-
 # Encrypt / Decrypt credentials data
 export SYNCMASTER__ENCRYPTION__CRYPTO_KEY=UBgPTioFrtH2unlC4XFDiGf5sYfzbdSf_VgiUSaQc94=
 
-# Postgres
-export SYNCMASTER__DATABASE__URL=postgresql+asyncpg://syncmaster:changeme@localhost:5432/syncmaster
+# Scheduler settings
+export SYNCMASTER__SCHEDULER__TRANSFER_FETCHING_TIMEOUT_SECONDS=200
 
 # Keycloack Auth
 export SYNCMASTER__AUTH__KEYCLOAK__SERVER_URL=http://keycloak:8080
@@ -35,9 +38,6 @@ export SYNCMASTER__AUTH__PROVIDER=syncmaster.backend.providers.auth.keycloak_pro
 export SYNCMASTER__AUTH__PROVIDER=syncmaster.backend.providers.auth.dummy_provider.DummyAuthProvider
 export SYNCMASTER__AUTH__ACCESS_TOKEN__SECRET_KEY=secret
 
-# RabbitMQ
-export SYNCMASTER__BROKER__URL=amqp://guest:guest@localhost:5672/
-
 export TEST_S3_HOST_FOR_CONFTEST=localhost
 export TEST_S3_PORT_FOR_CONFTEST=9010
 export TEST_S3_HOST_FOR_WORKER=test-s3

diff --git a/docs/_static/architecture.png b/docs/_static/architecture.png
diff --git a/docs/backend/auth/index.rst b/docs/backend/auth/index.rst
@@ -12,7 +12,7 @@ Syncmaster supports different auth provider implementations. You can change impl
     :caption: Auth providers
 
     dummy
-    keycloak
+    keycloak/index
 
 .. toctree::
     :maxdepth: 2

diff --git a/docs/backend/auth/keycloak/images/keycloak-client-authentication.png b/docs/backend/auth/keycloak/images/keycloak-client-authentication.png
diff --git a/docs/backend/auth/keycloak/images/keycloak-client-redirect_uri.png b/docs/backend/auth/keycloak/images/keycloak-client-redirect_uri.png
diff --git a/docs/backend/auth/keycloak/images/keycloak-client-secret.png b/docs/backend/auth/keycloak/images/keycloak-client-secret.png
diff --git a/docs/backend/auth/keycloak/images/keycloak-login.png b/docs/backend/auth/keycloak/images/keycloak-login.png
diff --git a/docs/backend/auth/keycloak/images/keycloak-new-client.png b/docs/backend/auth/keycloak/images/keycloak-new-client.png
diff --git a/docs/backend/auth/keycloak/images/keycloak-new-client_name.png b/docs/backend/auth/keycloak/images/keycloak-new-client_name.png
diff --git a/docs/backend/auth/keycloak/images/keycloak-new-realm.png b/docs/backend/auth/keycloak/images/keycloak-new-realm.png
diff --git a/docs/backend/auth/keycloak/images/keycloak-new-realm_name.png b/docs/backend/auth/keycloak/images/keycloak-new-realm_name.png
diff --git a/docs/backend/auth/keycloak.rst → docs/backend/auth/keycloak/index.rst b/docs/backend/auth/keycloak.rst → docs/backend/auth/keycloak/index.rst
@@ -1,17 +1,18 @@
-.. _backend-auth-ldap:
+.. _keycloak-auth-provider:
 
 KeyCloak Auth provider
-==================
+======================
 
 Description
 -----------
+Keycloak auth provider uses `python-keycloak <https://pypi.org/project/python-keycloak/>`_ library to interact with Keycloak server. During the authentication process,
+KeycloakAuthProvider redirects user to Keycloak authentication page.
 
-TODO:
+After successful authentication, Keycloak redirects user back to Syncmaster with authorization code.
+Then KeycloakAuthProvider exchanges authorization code for an access token and uses it to get user information from Keycloak server.
+If user is not found in Syncmaster database, KeycloakAuthProvider creates it. Finally, KeycloakAuthProvider returns user with access token.
 
-Strategies
-----------
-
-TODO:
+You can follow interaction schema below.
 
 Interaction schema
 ------------------
@@ -76,6 +77,13 @@ Interaction schema
 Basic configuration
 -------------------
 
-.. autopydantic_model:: syncmaster.settings.auth.keycloak.KeycloakProviderSettings
-.. autopydantic_model:: syncmaster.settings.auth.jwt.JWTSettings
+.. autopydantic_model:: syncmaster.backend.settings.auth.keycloak.KeycloakAuthProviderSettings
+.. autopydantic_model:: syncmaster.backend.settings.auth.keycloak.KeycloakSettings
+.. autopydantic_model:: syncmaster.backend.settings.auth.jwt.JWTSettings
+
+.. toctree::
+    :maxdepth: 1
+    :caption: Keycloak
+    :hidden:
 
+    local_installation
diff --git a/docs/backend/auth/keycloak/local_installation.rst b/docs/backend/auth/keycloak/local_installation.rst
@@ -0,0 +1,121 @@
+Local installation and testing
+------------------------------
+
+You can test Keycloak auth locally with docker compose:
+
+
+.. code-block:: console
+
+    $ docker compose -f docker-compose.test.yml up keycloak -d
+
+
+Authorize in keycloak
+~~~~~~~~~~~~~~~~~~~~~
+
+At first, you have to go to `http://localhost:8080/admin <http://localhost:8080/admin>`_ and login via login: `admin`, password: `admin` (by default) to create realms.
+
+.. image:: images/keycloak-login.png
+   :width: 400px
+   :align: center
+
+
+Create new realm
+~~~~~~~~~~~~~~~~
+
+.. image:: images/keycloak-new-realm.png
+   :width: 400px
+   :align: center
+
+
+Create new realm name
+~~~~~~~~~~~~~~~~~~~~~
+
+Pass realm name value. Then pass it to `SYNCMASTER__AUTH__KEYCLOAK__REALM_NAME` environment variable:
+
+.. code-block:: console
+
+    $ export SYNCMASTER__AUTH__KEYCLOAK__REALM_NAME=fastapi_realm  # as on screen below
+
+.. image:: images/keycloak-new-realm_name.png
+   :width: 400px
+   :align: center
+
+
+Create new client
+~~~~~~~~~~~~~~~~~
+
+.. image:: images/keycloak-new-client.png
+   :width: 400px
+   :align: center
+
+
+Create new client name
+~~~~~~~~~~~~~~~~~~~~~~
+
+In created realm pass client name value. Then pass it to `SYNCMASTER__AUTH__KEYCLOAK__CLIENT_ID` environment variable:
+
+.. code-block:: console
+
+    $ export SYNCMASTER__AUTH__KEYCLOAK__CLIENT_ID=fastapi_client  # as on screen below
+
+.. image:: images/keycloak-new-client_name.png
+   :width: 400px
+   :align: center
+
+
+Set ``client_authentication`` **ON** to receive client_secret
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+.. image:: images/keycloak-client-authentication.png
+   :width: 400px
+   :align: center
+
+Configure Redirect URI
+~~~~~~~~~~~~~~~~~~~~~~
+
+To configure the redirect URI where the browser will redirect to exchange the code provided from Keycloak for an access token, set the `SYNCMASTER__AUTH__KEYCLOAK__REDIRECT_URI` environment variable. The default value for local development is `http://localhost:8000/auth/callback`.
+
+.. code-block:: console
+
+    $ export SYNCMASTER__AUTH__KEYCLOAK__REDIRECT_URI=http://localhost:8000/auth/callback
+
+Configure the client redirect URI
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Ensure that this URI is also configured as a valid redirect URI in your Keycloak client settings. This allows the browser to redirect to your application after the user successfully authenticates with Keycloak.
+
+.. image:: images/keycloak-client-redirect_uri.png
+   :width: 400px
+   :align: center
+
+Configure the client secret
+~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Now go to **Credentials** tab and add the client secret to the `SYNCMASTER__AUTH__KEYCLOAK__CLIENT_SECRET` environment variable:
+
+.. code-block:: console
+
+    $ export SYNCMASTER__AUTH__KEYCLOAK__CLIENT_SECRET=6x6gn8uJdWSBmP8FqbNRSoGdvaoaFeez   # as on screen below
+
+.. image:: images/keycloak-client-secret.png
+   :width: 400px
+   :align: center
+
+Now you can use create users in this realms, check `keycloak documentation <https://www.keycloak.org/docs/latest/server_admin/#assembly-managing-users_server_administration_guide>`_ on how to manage users creation.
+
+ENVIRONMENT VARIABLES
+~~~~~~~~~~~~~~~~~~~~~
+
+After this you can user `KeycloakAuthProvider` in your application with provided environment variables:
+
+
+.. code-block:: console
+
+    $ export SYNCMASTER__AUTH__KEYCLOAK__SERVER_URL=http://keycloak:8080
+    $ export SYNCMASTER__AUTH__KEYCLOAK__REDIRECT_URI=http://localhost:8000/auth/callback
+    $ export SYNCMASTER__AUTH__KEYCLOAK__REALM_NAME=fastapi_realm
+    $ export SYNCMASTER__AUTH__KEYCLOAK__CLIENT_ID=fastapi_client
+    $ export SYNCMASTER__AUTH__KEYCLOAK__CLIENT_SECRET=6x6gn8uJdWSBmP8FqbNRSoGdvaoaFeez
+    $ export SYNCMASTER__AUTH__KEYCLOAK__SCOPE=email
+    $ export SYNCMASTER__AUTH__KEYCLOAK__VERIFY_SSL=False
+    $ export SYNCMASTER__AUTH__PROVIDER=syncmaster.backend.providers.auth.keycloak_provider.KeycloakAuthProvider
diff --git a/docs/backend/install.rst b/docs/backend/install.rst
@@ -79,15 +79,11 @@ Available *extras* are:
 Run database
 ~~~~~~~~~~~~
 
-Start Postgres instance somewhere, and set up environment variables:
+Start Postgres instance somewhere, and set up environment variable:
 
 .. code-block:: bash
 
-    POSTGRES_HOST=localhost
-    POSTGRES_PORT=5432
-    POSTGRES_DB=postgres
-    POSTGRES_USER=user
-    POSTGRES_PASSWORD=password
+    SYNCMASTER__DATABASE__URL=postgresql+asyncpg://syncmaster:changeme@db:5432/syncmaster
 
 You can use virtually any database supported by `SQLAlchemy <https://docs.sqlalchemy.org/en/20/core/engines.html#database-urls>`_,
 but the only one we really tested is Postgres.
@@ -111,33 +107,11 @@ options and commands are just the same.
 Run RabbitMQ
 ~~~~~~~~~~~~
 
-Start RabbitMQ instance somewhere, and set up environment variables:
+Start RabbitMQ instance somewhere, and set up environment variable:
 
 .. code-block:: bash
 
-    RABBITMQ_HOST=somehost
-    RABBITMQ_PORT=5672
-    RABBITMQ_USER=user
-    RABBITMQ_PASSWORD=password
-
-Run worker
-~~~~~~~~~~
-
-.. note::
-
-    Before starting the worker you need to create a queue.
-    The queue is created by sending a post request to ``/queues`` endpoint (See Swagger doc for details).
-
-to start the worker you need to run the command
-
-.. code-block:: console
-
-    $ celery -A syncmaster.worker.config.celery worker --loglevel=info --max-tasks-per-child=1 -Q queue_name
-
-.. note::
-
-    The specified celery options are given as an example, you can specify other options you need.
-
+    SYNCMASTER__BROKER__URL=amqp://guest:guest@rabbitmq:5672/
 
 Run backend
 ~~~~~~~~~~~

diff --git a/docs/design/entities.rst b/docs/design/entities.rst
@@ -121,7 +121,7 @@ Example:
         "started_at": "2024-01-19T16:30:07+03:00",
         "ended_at": null,
         "status": "STARTED",
-        "log_url": "https://kinaba.url/...",
+        "log_url": "https://kibana.url/...",
         "transfer_dump": {
             "transfer object JSON"
         },

diff --git a/docs/index.rst b/docs/index.rst
@@ -17,15 +17,17 @@
     backend/install
     backend/architecture
     backend/auth/index
-    backend/openapi
     backend/configuration/index
+    backend/openapi
 
 
 .. toctree::
     :maxdepth: 2
     :caption: Worker
     :hidden:
 
+    worker/start_worker
+    worker/log_url
     worker/configuration/index
 
 
@@ -34,6 +36,7 @@
     :caption: Scheduler
     :hidden:
 
+    scheduler/start_scheduler
     scheduler/configuration/index
 
 .. toctree::

diff --git a/docs/scheduler/start_scheduler.rst b/docs/scheduler/start_scheduler.rst
@@ -0,0 +1,41 @@
+Starting the Scheduler
+======================
+
+
+With docker
+-----------
+
+Installation process
+~~~~~~~~~~~~~~~~~~~~
+
+Docker will download worker image of syncmaster scheduler (same as backend image) & broker, and run them.
+Options can be set via ``.env`` file or ``environment`` section in ``docker-compose.yml``
+
+.. dropdown:: ``docker-compose.yml``
+
+    .. literalinclude:: ../../docker-compose.yml
+       :emphasize-lines: 68-83
+
+.. dropdown:: ``.env.docker``
+
+    .. literalinclude:: ../../.env.docker
+       :emphasize-lines: 11-25
+
+To start the worker container you need to run the command:
+
+.. code-block:: bash
+
+    docker compose up scheduler -d --wait --wait-timeout 200
+
+
+
+Without docker
+--------------
+
+To start the scheduler you need to run the command
+
+.. code-block:: bash
+
+    python -m syncmaster.scheduler
+
+