Add tt migrations (#4499)

Resolves tarantool/enterprise_doc#269, tarantool/enterprise_doc#277, tarantool/enterprise_doc#278, tarantool/enterprise_doc#279

Co-authored-by: Georgy Moiseev <moiseev.georgii@gmail.com>
Co-authored-by: Kseniia Antonova <73473519+xuniq@users.noreply.github.com>

doc/code_snippets/snippets/migrations/README.md

@@ -0,0 +1,4 @@
+# Centralized migrations with tt
+
+Sample applications demonstrating how to use the centralized migration mechanism
+for Tarantool EE clusters via the tt utility. Learn more at [Centralized configuration storages](https://www.tarantool.io/en/doc/latest/platform/https://www.tarantool.io/en/doc/latest/platform/ddl_dml/migrations/).

doc/code_snippets/snippets/migrations/etcd_setup.sh

@@ -0,0 +1,12 @@
+#!/usr/bin/env bash
+
+# 1. Remove the 'default.etcd' directory to reset etcd to initial state.
+# 2. Start etcd by executing the 'etcd' command.
+# 3. Execute this script to enable authentication.
+
+etcdctl user add root:topsecret
+etcdctl role add app_config_manager
+etcdctl role grant-permission app_config_manager --prefix=true readwrite /myapp/
+etcdctl user add app_user:config_pass
+etcdctl user grant-role app_user app_config_manager
+etcdctl auth enable

doc/code_snippets/snippets/migrations/instances.enabled/myapp/config.yaml

@@ -0,0 +1,10 @@
+config:
+  etcd:
+    endpoints:
+    - http://localhost:2379
+    prefix: /myapp/
+    username: app_user
+    password: config_pass
+    http:
+      request:
+        timeout: 3

doc/code_snippets/snippets/migrations/instances.enabled/myapp/instances-3-storages.yml

@@ -0,0 +1,7 @@
+router-001-a:
+storage-001-a:
+storage-001-b:
+storage-002-a:
+storage-002-b:
+storage-003-a:
+storage-003-b:

doc/code_snippets/snippets/migrations/instances.enabled/myapp/instances.yml

@@ -0,0 +1,5 @@
+router-001-a:
+storage-001-a:
+storage-001-b:
+storage-002-a:
+storage-002-b:

doc/code_snippets/snippets/migrations/instances.enabled/myapp/myapp-scm-1.rockspec

@@ -0,0 +1,14 @@
+package = 'myapp'
+version = 'scm-1'
+
+source  = {
+    url = '/dev/null',
+}
+
+dependencies = {
+    'crud == 1.5.2',
+}
+
+build = {
+    type = 'none';
+}

doc/code_snippets/snippets/migrations/instances.enabled/myapp/source-3-storages.yaml

@@ -0,0 +1,88 @@
+credentials:
+  users:
+    client:
+      password: 'secret'
+      roles: [super]
+    replicator:
+      password: 'secret'
+      roles: [replication]
+    storage:
+      password: 'secret'
+      roles: [sharding]
+
+iproto:
+  advertise:
+    peer:
+      login: replicator
+    sharding:
+      login: storage
+
+sharding:
+  bucket_count: 3000
+
+groups:
+  routers:
+    sharding:
+      roles: [router]
+    roles: [roles.crud-router]
+    replicasets:
+      router-001:
+        instances:
+          router-001-a:
+            iproto:
+              listen:
+              - uri: localhost:3301
+              advertise:
+                client: localhost:3301
+  storages:
+    sharding:
+      roles: [storage]
+    roles: [roles.crud-storage]
+    replication:
+      failover: manual
+    replicasets:
+      storage-001:
+        leader: storage-001-a
+        instances:
+          storage-001-a:
+            iproto:
+              listen:
+                - uri: localhost:3302
+              advertise:
+                client: localhost:3302
+          storage-001-b:
+            iproto:
+              listen:
+              - uri: localhost:3303
+              advertise:
+                client: localhost:3303
+      storage-002:
+        leader: storage-002-a
+        instances:
+          storage-002-a:
+            iproto:
+              listen:
+              - uri: localhost:3304
+              advertise:
+                client: localhost:3304
+          storage-002-b:
+            iproto:
+              listen:
+              - uri: localhost:3305
+              advertise:
+                client: localhost:3305
+      storage-003:
+        leader: storage-003-a
+        instances:
+          storage-003-a:
+            iproto:
+              listen:
+              - uri: localhost:3306
+              advertise:
+                client: localhost:3306
+          storage-003-b:
+            iproto:
+              listen:
+              - uri: localhost:3307
+              advertise:
+                client: localhost:3307

doc/code_snippets/snippets/migrations/instances.enabled/myapp/source.yaml

@@ -0,0 +1,73 @@
+credentials:
+  users:
+    client:
+      password: 'secret'
+      roles: [super]
+    replicator:
+      password: 'secret'
+      roles: [replication]
+    storage:
+      password: 'secret'
+      roles: [sharding]
+
+iproto:
+  advertise:
+    peer:
+      login: replicator
+    sharding:
+      login: storage
+
+sharding:
+  bucket_count: 3000
+
+groups:
+  routers:
+    sharding:
+      roles: [router]
+    roles: [roles.crud-router]
+    replicasets:
+      router-001:
+        instances:
+          router-001-a:
+            iproto:
+              listen:
+              - uri: localhost:3301
+              advertise:
+                client: localhost:3301
+  storages:
+    sharding:
+      roles: [storage]
+    roles: [roles.crud-storage]
+    replication:
+      failover: manual
+    replicasets:
+      storage-001:
+        leader: storage-001-a
+        instances:
+          storage-001-a:
+            iproto:
+              listen:
+                - uri: localhost:3302
+              advertise:
+                client: localhost:3302
+          storage-001-b:
+            iproto:
+              listen:
+              - uri: localhost:3303
+              advertise:
+                client: localhost:3303
+      storage-002:
+        leader: storage-002-a
+        instances:
+          storage-002-a:
+            iproto:
+              listen:
+              - uri: localhost:3304
+              advertise:
+                client: localhost:3304
+          storage-002-b:
+            iproto:
+              listen:
+              - uri: localhost:3305
+              advertise:
+                client: localhost:3305

doc/code_snippets/snippets/migrations/migrations/scenario/000001_create_writers_space.lua

@@ -0,0 +1,23 @@
+local helpers = require('tt-migrations.helpers')
+
+local function apply_scenario()
+    local space = box.schema.space.create('writers')
+
+    space:format({
+        {name = 'id', type = 'number'},
+        {name = 'bucket_id', type = 'number'},
+        {name = 'name', type = 'string'},
+        {name = 'age', type = 'number'},
+    })
+
+    space:create_index('primary', {parts = {'id'}})
+    space:create_index('bucket_id', {parts = {'bucket_id'}})
+
+    helpers.register_sharding_key('writers', {'id'})
+end
+
+return {
+    apply = {
+        scenario = apply_scenario,
+    },
+}

doc/code_snippets/snippets/migrations/migrations/scenario/000002_create_writers_index.lua

@@ -0,0 +1,11 @@
+local function apply_scenario()
+    local space = box.space['writers']
+
+    space:create_index('age', {parts = {'age'}})
+end
+
+return {
+    apply = {
+        scenario = apply_scenario,
+    },
+}

doc/code_snippets/snippets/migrations/migrations/scenario/000003_alter_writers_space.lua

@@ -0,0 +1,48 @@
+local function apply_scenario()
+    local space = box.space['writers']
+    local new_format = {
+        {name = 'id', type = 'number'},
+        {name = 'bucket_id', type = 'number'},
+        {name = 'first_name', type = 'string'},
+        {name = 'last_name', type = 'string'},
+        {name = 'age', type = 'number'},
+    }
+    box.space.writers.index.age:drop()
+
+    box.schema.func.create('_writers_split_name', {
+        language = 'lua',
+        is_deterministic = true,
+        body = [[
+        function(t)
+            local name = t[3]
+
+            local split_data = {}
+            local split_regex = '([^%s]+)'
+            for v in string.gmatch(name, split_regex) do
+                table.insert(split_data, v)
+            end
+
+            local first_name = split_data[1]
+            assert(first_name ~= nil)
+
+            local last_name = split_data[2]
+            assert(last_name ~= nil)
+
+            return {t[1], t[2], first_name, last_name, t[4]}
+        end
+        ]],
+    })
+
+    local future = space:upgrade({
+        func = '_writers_split_name',
+        format = new_format,
+    })
+
+    future:wait()
+end
+
+return {
+    apply = {
+        scenario = apply_scenario,
+    },
+}

doc/code_snippets/snippets/migrations/tt.yaml

@@ -0,0 +1,54 @@
+modules:
+  # Directory where the external modules are stored.
+  directory: modules
+
+env:
+  # Restart instance on failure.
+  restart_on_failure: false
+
+  # Directory that stores binary files.
+  bin_dir: bin
+
+  # Directory that stores Tarantool header files.
+  inc_dir: include
+
+  # Path to directory that stores all applications.
+  # The directory can also contain symbolic links to applications.
+  instances_enabled: instances.enabled
+
+  # Tarantoolctl artifacts layout compatibility: if set to true tt will not create application
+  # sub-directories for control socket, pid files, log files, etc.. Data files (wal, vinyl,
+  # snap) and multi-instance applications are not affected by this option.
+  tarantoolctl_layout: false
+
+app:
+  # Directory that stores various instance runtime
+  # artifacts like console socket, PID file, etc.
+  run_dir: var/run
+
+  # Directory that stores log files.
+  log_dir: var/log
+
+  # Directory where write-ahead log (.xlog) files are stored.
+  wal_dir: var/lib
+
+  # Directory where memtx stores snapshot (.snap) files.
+  memtx_dir: var/lib
+
+  # Directory where vinyl files or subdirectories will be stored.
+  vinyl_dir: var/lib
+
+# Path to file with credentials for downloading Tarantool Enterprise Edition.
+# credential_path: /path/to/file
+ee:
+  credential_path: 
+
+templates:
+  # The path to templates search directory.
+  - path: templates
+
+repo:
+  # Directory where local rocks files could be found.
+  rocks: 
+  # Directory that stores installation files.
+  distfiles: distfiles

doc/platform/ddl_dml/index.rst

@@ -32,7 +32,7 @@ This section contains guides on performing data operations in Tarantool.
     value_store
     schema_desc
     operations
-    migrations
+    migrations/index
     read_views
     sql/index