Rakefile

require 'json'
require 'date'

# tools versions
NODE_VER = 'v18.16.0'
OPENAPI_GENERATOR_VER = '6.6.0'
HELM_VER = 'v3.12.0'
DOCKER_COMPOSE_VER = '2.2.2'
PULUMI_VER = 'v3.68.0'  # 'v3.26.1'

# Check host OS
UNAME=`uname -s`

case UNAME.rstrip
  when "Darwin"
    sysname = "Darwin"
    sysnamel = "darwin"
  when "Linux"
    sysname = "Linux"
    sysnamel = "linux"
  else
    puts "ERROR: Unknown/unsupported OS: %s" % UNAME
    fail
  end

TOOLS_DIR = File.expand_path('tools')
ENV['PATH'] = "#{TOOLS_DIR}/node-#{NODE_VER}-#{sysnamel}-x64/bin:#{ENV['PATH']}"
NPX = "#{TOOLS_DIR}/node-#{NODE_VER}-#{sysnamel}-x64/bin/npx"
NG = File.expand_path('ui/node_modules/.bin/ng')
OPENAPI_GENERATOR = "#{TOOLS_DIR}/swagger-codegen-cli-#{OPENAPI_GENERATOR_VER}.jar"
SWAGGER_FILE = File.expand_path("server/kraken/server/swagger.yml")
HELM = "#{TOOLS_DIR}/helm"
PULUMI = "#{TOOLS_DIR}/pulumi-#{PULUMI_VER}/pulumi"

DOCKER_COMPOSE = "#{TOOLS_DIR}/docker-compose-#{DOCKER_COMPOSE_VER}"
ENV['DOCKER_BUILDKIT']='1'
ENV['COMPOSE_DOCKER_CLI_BUILD']='1'

kk_ver = ENV['kk_ver'] || '0.0.0'
ENV['KRAKEN_VERSION'] = kk_ver
KRAKEN_VERSION_FILE = File.expand_path("kraken-version-#{kk_ver}.txt")

now = Time.new
kk_stamp = now.strftime('%Y%m%d%H%M')

helm_dest = '.'
if ENV['helm_dest']
  helm_dest = ENV['helm_dest']
end

LOCALHOST_IP=ENV['LOCALHOST_IP'] || '192.168.68.121'
CLICKHOUSE_ADDR="#{LOCALHOST_IP}:9001"
MINIO_ADDR="#{LOCALHOST_IP}:9999"
MINIO_ROOT_USER='UFSEHRCFU4ACUEWHCHWU'
MINIO_ROOT_PASSWORD='HICSHuhIIUhiuhMIUHIUhGFfUHugy6fGJuyyfiGY'

KK_SERVER_TGZ="krakenci_server-#{kk_ver}.tar.gz"
KK_SERVER_TGZ_PATH="server/dist/#{KK_SERVER_TGZ}"
KK_AGENT_TGZ="krakenci_agent-#{kk_ver}.tar.gz"
KK_AGENT_TGZ_PATH="agent/#{KK_AGENT_TGZ}"
KK_CLIENT_TGZ="krakenci_client-#{kk_ver}.tar.gz"
KK_CLIENT_TGZ_PATH="client/dist/#{KK_CLIENT_TGZ}"
KK_WEB_UI_TGZ="krakenci_ui-#{kk_ver}.tar.gz"
KK_WEB_UI_TGZ_PATH="ui/dist/#{KK_WEB_UI_TGZ}"
KK_CH_PROXY_TGZ="clickhouse-proxy-#{kk_ver}.tar.gz"

file DOCKER_COMPOSE do
  sh "mkdir -p #{TOOLS_DIR}"
  sh "wget -nv https://github.com/docker/compose/releases/download/v#{DOCKER_COMPOSE_VER}/docker-compose-#{sysname}-x86_64 -O #{DOCKER_COMPOSE}"
  sh "chmod a+x #{DOCKER_COMPOSE}"
end

# prepare env
file './venv/bin/python3' do
  sh 'python3 -m venv venv'
  sh './venv/bin/pip install -U pip'
  sh './venv/bin/pip install -r requirements.txt'
end

file './agent/venv/bin/python3' do
  sh 'python3 -m venv agent/venv'
  sh './agent/venv/bin/pip install -U pip'
  sh './agent/venv/bin/pip install -r agent/reqs-ut.txt'
end

task :prepare_env => ['./venv/bin/python3', './agent/venv/bin/python3', 'server/kraken/version.py'] do
  sh 'sudo DEBIAN_FRONTEND=noninteractive apt-get install -y openjdk-17-jre-headless python3-venv libpq-dev gcc libpython3-dev libldap-dev libsasl2-dev'
  sh 'cp README.md server/'
  sh 'cd server && ../venv/bin/poetry install'
  sh 'rm server/README.md'
  sh 'cd client && ../venv/bin/poetry install'
end

# UI
task :gen_client => [OPENAPI_GENERATOR, SWAGGER_FILE] do
  Dir.chdir('ui') do
    sh "java -jar #{OPENAPI_GENERATOR} generate  -g typescript-angular -i #{SWAGGER_FILE} -o src/app/backend --additional-properties snapshot=true,ngVersion=16.1.2,modelPropertyNaming=snake_case"
  end
end

file OPENAPI_GENERATOR do
  sh "mkdir -p #{TOOLS_DIR}"
  sh "wget -nv https://repo1.maven.org/maven2/org/openapitools/openapi-generator-cli/#{OPENAPI_GENERATOR_VER}/openapi-generator-cli-#{OPENAPI_GENERATOR_VER}.jar -O #{OPENAPI_GENERATOR}"
end

file NPX do
  sh "mkdir -p #{TOOLS_DIR}"
  Dir.chdir(TOOLS_DIR) do
    sh "wget -nv https://nodejs.org/dist/#{NODE_VER}/node-#{NODE_VER}-#{sysnamel}-x64.tar.xz -O #{TOOLS_DIR}/node.tar.xz"
    sh "tar -Jxf node.tar.xz"
  end
end

file NG => NPX do
  Dir.chdir('ui') do
    sh 'NG_CLI_ANALYTICS=false npm install'
  end
end

task :build_ui => KK_WEB_UI_TGZ_PATH
file KK_WEB_UI_TGZ_PATH => [NG, :gen_client] do
  Dir.chdir('ui') do
    sh "sed -e 's/0\.0\.0/#{kk_ver}/g' src/environments/environment.prod.ts.in > src/environments/environment.prod.ts"
    sh 'npx ng build --configuration production'
    sh 'cp nginx.conf dist/kraken'
    sh 'sed -i -e "s/\${DOLLAR}/\$/g" dist/kraken/nginx.conf'
    Dir.chdir('dist') do
      sh "tar --transform 's,^kraken,krakenci_ui-#{kk_ver},' -zcf #{KK_WEB_UI_TGZ} kraken/"
    end
  end
  sh "tar -ztvf #{KK_WEB_UI_TGZ_PATH}"
end

task :serve_ui => [NG, :gen_client] do
  Dir.chdir('ui') do
    sh 'npx ng serve --host 0.0.0.0 --disable-host-check --proxy-config proxy.conf.json'
  end
end

task :lint_ui => [NG, :gen_client] do
  Dir.chdir('ui') do
    sh 'npm ci'
    sh 'npx ng lint'
    sh 'npx prettier --config .prettierrc --check \'**/*\''
  end
end

task :fix_ui => [NG, :gen_client] do
  Dir.chdir('ui') do
    sh 'npx ng lint --fix'
    sh 'npx prettier --config .prettierrc --write \'**/*\''
  end
end

task :lint_py => ['./venv/bin/python3', 'server/kraken/version.py', './agent/venv/bin/python3'] do
  Dir.chdir('server') do
    sh '../venv/bin/poetry install'
    sh '../venv/bin/poetry run pylint --rcfile ../pylint.rc kraken || true'
  end
  Dir.chdir('client') do
    sh '../venv/bin/poetry run pylint --rcfile ../pylint.rc kraken || true'
  end
  Dir.chdir('agent') do
    sh './venv/bin/pip install -r reqs-ut.txt'
    sh './venv/bin/pylint --rcfile ../pylint.rc kraken || true'
  end
end


# BACKEND


# TODO TODEL
#file './server/venv/bin/python3' do
#  sh 'python3 -m venv server/venv'
#  sh './server/venv/bin/pip install -U pip'
#end

# TODO TODEL
#def setup_py_develop
#  Dir.chdir('server') do
#    sh './venv/bin/pip install -r requirements.txt'
#    sh './venv/bin/python3 setup.py develop --upgrade'
#  end
#end

file KRAKEN_VERSION_FILE do
  sh 'rm -f kraken-version-*.txt'
  sh "touch #{KRAKEN_VERSION_FILE}"
end

file 'server/kraken/version.py' => KRAKEN_VERSION_FILE do
  sh "echo \"version = '#{kk_ver}'\" > server/kraken/version.py"
end

file 'client/kraken/client/version.py' => KRAKEN_VERSION_FILE do
  sh "echo \"version = '#{kk_ver}'\" > client/kraken/client/version.py"
end

task :run_server => 'server/kraken/version.py' do
  sh 'cp dot.env .env'
  Dir.chdir('server') do
    sh "KRAKEN_CLICKHOUSE_ADDR=#{CLICKHOUSE_ADDR} KRAKEN_MINIO_ADDR=#{MINIO_ADDR} MINIO_ROOT_USER='#{MINIO_ROOT_USER}' MINIO_ROOT_PASSWORD='#{MINIO_ROOT_PASSWORD}' ../venv/bin/poetry run python -m kraken.server.server"
  end
end

task :run_scheduler => 'server/kraken/version.py' do
  Dir.chdir('server') do
    sh "KRAKEN_CLICKHOUSE_ADDR=#{CLICKHOUSE_ADDR} ../venv/bin/poetry run kkscheduler"
  end
end

file './agent/venv/bin/kkagent' => './agent/venv/bin/python3' do
  Dir.chdir('agent') do
    sh './venv/bin/pip install -r requirements.txt'
    sh './venv/bin/python3 setup.py develop --upgrade'
  end
end

task :run_agent => ['./agent/venv/bin/kkagent', :build_agent] do
  sh "rm -f #{KK_AGENT_TGZ_PATH}"
  Rake::Task["build_agent"].invoke
  sh 'rm -rf /opt/kraken/*'
  sh 'cp agent/kkagent agent/kktool /opt/kraken'
  sh "LANGUAGE=en_US:en LANG=en_US.UTF-8 LC_ALL=en_US.UTF-8 KRAKEN_CLICKHOUSE_ADDR=#{CLICKHOUSE_ADDR} /opt/kraken/kkagent run -d /tmp/kk-jobs -s http://localhost:8080 --no-update"
end

task :run_agent_in_docker do
  sh "rm -f #{KK_AGENT_TGZ_PATH}"
  Rake::Task["build_agent"].invoke
  Dir.chdir('agent') do
    sh 'docker build -f docker-agent.txt -t kkagent .'
  end
  sh "docker run --rm -ti  -v /var/run/docker.sock:/var/run/docker.sock -v /var/snap/lxd/common/lxd/unix.socket:/var/snap/lxd/common/lxd/unix.socket -v `pwd`/agent:/agent -e KRAKEN_AGENT_SLOT=7 -e KRAKEN_CLICKHOUSE_ADDR=#{CLICKHOUSE_ADDR} kkagent /opt/kraken/kkagent run -s http://#{LOCALHOST_IP}:8080"
end

task :run_agent_in_lxd_all do
#  Rake::Task["build_agent"].invoke
  Dir.chdir('agent') do
    systems = [
#      ['images:centos/8-Stream/amd64', 'c8s'],
#      ['images:centos/9-Stream/amd64', 'c9s'],
#      ['images:debian/10/amd64', 'd10'],
#      ['images:debian/11/amd64', 'd11'],
#      ['images:opensuse/15.3/amd64', 's153'],
#      ['images:ubuntu/focal/amd64', 'u20'],
#      ['images:ubuntu/jammy/amd64', 'u22'],
#      ['images:fedora/34/amd64', 'f34'],
#      ['images:fedora/35/amd64', 'f35'],
      ['images:rockylinux/8/amd64', 'rl8'],
    ]

#    sh 'lxc network delete kk-net || true'
#    sh 'lxc network create kk-net || true'
#    sh "sleep 10"
    systems.each do |sys, name|
      cntr_name = "kk-agent-#{name}"
      sh "lxc stop #{cntr_name} || true"
      sh "lxc delete #{cntr_name} || true"
      sh "lxc launch #{sys} #{cntr_name}"
      sh "lxc network attach kk-net #{cntr_name}"
      sh "sleep 5"
      if sys.include?('centos/7') or sys.include?('debian/buster')
        sh "lxc exec #{cntr_name} -- dhclient"
      end
      if sys.include?('centos') or sys.include?('rocky')
        sh "lxc exec #{cntr_name} -- yum install -y python39 sudo wget which"
      end
      if sys.include?('fedora')
        sh "lxc exec #{cntr_name} -- dnf install -y wget"
      end
      if sys.include?('debian')
        sh "lxc exec #{cntr_name} -- apt-get update"
        sh "lxc exec #{cntr_name} -- apt-get install -y curl python3 sudo wget"
      end
      if sys.include?('opensuse/15')
        sh "lxc exec #{cntr_name} -- zypper install -y curl wget python39 sudo system-group-wheel"
        sh "lxc exec #{cntr_name} -- ln -sf /usr/bin/python3.9 /usr/bin/python3"
      end
      if sys.include?('ubuntu')
        sh "lxc exec #{cntr_name} -- apt install -y wget"
      end
      #sh "lxc file push agent/kkagent #{cntr_name}/root/kkagent"
      #sh "lxc exec #{cntr_name} -- ./kkagent install -s http://#{LOCALHOST_IP}:8080"
      sh "lxc exec #{cntr_name} -- wget http://#{LOCALHOST_IP}:4200/bk/install/kraken-agent-install.sh"
      sh "lxc exec #{cntr_name} -- chmod a+x kraken-agent-install.sh"
      #sh "lxc exec #{cntr_name} -- bash -c 'LC_ALL=en_US.utf8 LANG=en_US.utf8 ./kraken-agent-install.sh'"
      sh "lxc exec #{cntr_name} -- ./kraken-agent-install.sh"
      sh "lxc exec #{cntr_name} -- journalctl -u kraken-agent.service"
      # sh "lxc exec #{cntr_name} -- journalctl -f -u kraken-agent.service'
    end
  end
end

task :run_agent_in_lxd do
  Rake::Task["build_agent"].invoke
  Dir.chdir('agent') do
    systems = [
      #['ubuntu:20.04', 'u20'],
      ['images:ubuntu/mantic/amd64', 'u2310'],
    ]

    sh 'lxc network delete kk-net || true'
    sh 'lxc network create kk-net || true'
    systems.each do |sys, name|
      cntr_name = "kk-agent-#{name}"
      sh "lxc stop #{cntr_name} || true"
      sh "lxc delete #{cntr_name} || true"
      sh "lxc launch #{sys} #{cntr_name}"
      sh "lxc config set #{cntr_name} security.nesting true"
      sh "lxc network attach kk-net #{cntr_name}"
      sh "lxc exec #{cntr_name} -- sleep 5"
      if sys.include?('centos/7') or sys.include?('debian/buster')
        sh "lxc exec #{cntr_name} -- dhclient"
      end
      if sys.include?('centos')
        sh "lxc exec #{cntr_name} -- yum install -y python3 sudo"
      end
      if sys.include?('debian')
        sh "lxc exec #{cntr_name} -- apt-get update"
        sh "lxc exec #{cntr_name} -- apt-get install -y curl python3 sudo"
      end
      if sys.include?('opensuse/15.2')
        sh "lxc exec #{cntr_name} -- zypper install -y curl python3 sudo system-group-wheel"
      end

      sh "lxc exec #{cntr_name} -- apt-get update"
      sh "lxc exec #{cntr_name} -- apt-get install -y python3-docker docker.io wget python3.12"
      sh "lxc exec #{cntr_name} -- ln -sf /usr/bin/python3.12 /usr/bin/python3"

      sh "lxc exec #{cntr_name} -- wget http://#{LOCALHOST_IP}:4200/bk/install/kraken-agent-install.sh"
      sh "lxc exec #{cntr_name} -- chmod a+x kraken-agent-install.sh"
      sh "lxc exec #{cntr_name} -- ./kraken-agent-install.sh"
      sh "lxc exec #{cntr_name} -- journalctl -f -u kraken-agent.service"
    end
  end
end

task :run_rq => './server/kraken/version.py' do
  sh 'cp dot.env .env'
  Dir.chdir('server') do
    sh "KRAKEN_CLICKHOUSE_ADDR=#{CLICKHOUSE_ADDR} KRAKEN_MINIO_ADDR=#{MINIO_ADDR} MINIO_ROOT_USER='#{MINIO_ROOT_USER}' MINIO_ROOT_PASSWORD='#{MINIO_ROOT_PASSWORD}' ../venv/bin/poetry run kkrq"
  end
end

task :run_qneck => './server/kraken/version.py' do
  Dir.chdir('server') do
    sh "KRAKEN_CLICKHOUSE_ADDR=#{CLICKHOUSE_ADDR} KRAKEN_MINIO_ADDR=#{MINIO_ADDR} MINIO_ROOT_USER='#{MINIO_ROOT_USER}' MINIO_ROOT_PASSWORD='#{MINIO_ROOT_PASSWORD}' ../venv/bin/poetry run kkqneck"
  end
end

task :run_planner => 'server/kraken/version.py' do
  Dir.chdir('server') do
    sh "KRAKEN_CLICKHOUSE_ADDR=#{CLICKHOUSE_ADDR} ../venv/bin/poetry run kkplanner"
  end
end

task :run_watchdog => 'server/kraken/version.py' do
  Dir.chdir('server') do
    sh "KRAKEN_CLICKHOUSE_ADDR=#{CLICKHOUSE_ADDR} ../venv/bin/poetry run kkwatchdog"
  end
end

file './venv/bin/shiv' => ['./venv/bin/python3', 'requirements.txt'] do
    sh './venv/bin/pip install -r requirements.txt'
end

agent_src = [
  'agent/kraken/agent/__init__.py',
  'agent/kraken/agent/agent.py',
  'agent/kraken/agent/config.py',
  'agent/kraken/agent/consts.py',
  'agent/kraken/agent/docker_run.py',
  'agent/kraken/agent/install.py',
  'agent/kraken/agent/jobber.py',
  'agent/kraken/agent/kraken_artifacts.py',
  'agent/kraken/agent/kraken_cache.py',
  'agent/kraken/agent/kraken_cloc.py',
  'agent/kraken/agent/kraken_data.py',
  'agent/kraken/agent/kraken_git.py',
  'agent/kraken/agent/kraken_gotest.py',
  'agent/kraken/agent/kraken_junit_collect.py',
  'agent/kraken/agent/kraken_nglint.py',
  'agent/kraken/agent/kraken_pylint.py',
  'agent/kraken/agent/kraken_pytest.py',
  'agent/kraken/agent/kraken_rndtest.py',
  'agent/kraken/agent/kraken_shell.py',
  'agent/kraken/agent/kraken_values_collect.py',
  'agent/kraken/agent/local_run.py',
  'agent/kraken/agent/logs.py',
  'agent/kraken/agent/lxd_run.py',
  'agent/kraken/agent/main.py',
  'agent/kraken/agent/miniobase.py',
  'agent/kraken/agent/server.py',
  'agent/kraken/agent/sshkey.py',
  'agent/kraken/agent/sysutils.py',
  'agent/kraken/agent/tool.py',
  'agent/kraken/agent/update.py',
  'agent/kraken/agent/utils.py',

  'agent/requirements.txt',
  'agent/reqs-tool.txt',
]

file 'agent/kraken/agent/consts.py' => 'server/kraken/server/consts.py' do
  sh 'cp server/kraken/server/consts.py agent/kraken/agent/'
end

file 'agent/kraken/agent/logs.py' => 'server/kraken/server/logs.py' do
  sh 'cp server/kraken/server/logs.py agent/kraken/agent/'
end

task :build_agent => KK_AGENT_TGZ_PATH
file KK_AGENT_TGZ_PATH => ['./venv/bin/python3', './venv/bin/shiv'] + agent_src do
  Dir.chdir('agent') do
    sh 'rm -rf dist-agent'
    sh '../venv/bin/pip install --target dist-agent -r requirements.txt'
    sh '../venv/bin/pip install --target dist-agent --upgrade .'
    sh "../venv/bin/shiv --site-packages dist-agent --compressed -p '/usr/bin/env python3' -o kkagent -c kkagent"
    sh 'rm -rf dist-tool'
    sh '../venv/bin/pip install --target dist-tool -r reqs-tool.txt'
    sh '../venv/bin/pip install --target dist-tool --upgrade .'
    sh "../venv/bin/shiv --site-packages dist-tool --compressed -p '/usr/bin/env python3' -o kktool -c kktool"
    sh "tar --transform 's,^,krakenci_agent-#{kk_ver}/,' -zcf #{KK_AGENT_TGZ} kkagent kktool"
  end
  sh "cp agent/kkagent agent/kktool server/"
  sh "tar -ztvf #{KK_AGENT_TGZ_PATH}"
end

file 'client/kraken/client/toolops.py' => 'server/kraken/server/toolops.py' do
  sh 'cp server/kraken/server/toolops.py client/kraken/client/'
end

task :build_client => KK_CLIENT_TGZ_PATH
file KK_CLIENT_TGZ_PATH => ['./venv/bin/python3', 'client/kraken/client/version.py', 'client/kraken/client/toolops.py']  do
  Dir.chdir('client') do
    sh 'rm -rf dist'
    sh 'rm -rf kraken/agent'
    sh 'mkdir kraken/agent'
    sh 'cp ../agent/kraken/agent/utils.py ../server/kraken/server/consts.py ../agent/kraken/agent/sysutils.py ../agent/kraken/agent/tool.py kraken/agent/'
    sh "../venv/bin/poetry build -f sdist -vvv --override-version #{kk_ver}"
  end
  sh "tar -ztvf #{KK_CLIENT_TGZ_PATH}"
end

task :publish_client => './venv/bin/python3' do
  Dir.chdir('client') do
    sh "../venv/bin/poetry publish -u __token__ -p #{ENV['PYPI_CLIENT_TOKEN']} --override-version #{kk_ver}"
  end
end

file 'server/nssm-2.24.zip' do
  sh 'wget -nv http://nssm.cc/release/nssm-2.24.zip -O server/nssm-2.24.zip'
end

task :build_server => KK_SERVER_TGZ_PATH
file KK_SERVER_TGZ_PATH => ['./venv/bin/python3', 'server/kraken/version.py', 'server/nssm-2.24.zip']  do
  Dir.chdir('server') do
    sh 'rm -rf dist'
    sh 'cp ../README.md .'
    sh "../venv/bin/poetry build -f sdist -vvv --override-version #{kk_ver}"
    sh 'rm README.md'
  end
  sh "tar -ztvf #{KK_SERVER_TGZ_PATH}"
end

task :publish_server => './venv/bin/python3' do
  Dir.chdir('server') do
    sh 'cp ../README.md .'
    sh "../venv/bin/poetry publish -u __token__ -p #{ENV['PYPI_SERVER_TOKEN']} --override-version #{kk_ver}"
    sh 'rm README.md'
  end
end

task :build_py => [:build_server, :build_agent, :build_client]

task :clean_backend do
  sh 'rm -rf venv'
  sh 'rm -rf server/venv'
  sh 'rm -rf agent/venv'
end

task :build_all => [:build_py, :build_ui]

task :server_ut do
  at_exit {
    sh "docker rm -f -v kk-ut-pgsql"
  }
  sh "docker run --name kk-ut-pgsql -d -p 15432:5432 -e POSTGRES_DB=kkut -e POSTGRES_USER=kkut -e POSTGRES_PASSWORD=kkut postgres:11 && sleep 5"

  ENV['POSTGRES_URL'] = "postgresql://kkut:kkut@localhost:15432/"
  Dir.chdir('server') do
    sh 'rm -f .coverage'
    sh "../venv/bin/poetry run pytest --ignore=./tests/example_tool --cov-branch --cov=. -s --log-level=info -r A -vv #{ENV['test']}"
    sh 'rm -rf htmlcov'
    sh '../venv/bin/poetry run coverage html -i --omit=./tests/*'
  end
end

task :agent_ut => ['./agent/venv/bin/python3'] do
  Dir.chdir('agent') do
    sh './venv/bin/pip install -r requirements.txt'
    sh './venv/bin/pip install -r reqs-ut.txt'
    sh "PYTHONPATH=. ./venv/bin/pytest --ignore=./dist-tool --ignore=./dist-agent --cov-branch --cov=. -s -r A -vv #{ENV['test']}"
    sh 'rm -rf htmlcov'
    sh './venv/bin/coverage html -i --omit=./tests/*'
  end
end

# DATABASE
DB_URL = "postgresql://kraken:kk123@localhost:5433/kraken"
task :db_up do
  Dir.chdir('server/kraken/migrations') do
    sh "KRAKEN_DB_URL=#{DB_URL} ../../../venv/bin/poetry run alembic -c alembic.ini upgrade head"
  end
end

task :db_down do
  Dir.chdir('server/kraken/migrations') do
    sh "KRAKEN_DB_URL=#{DB_URL} ../../../venv/bin/poetry run alembic -c alembic.ini downgrade -1"
  end
end

task :db_init do
  Dir.chdir('server/kraken/migrations') do
    sh "KRAKEN_DB_URL=#{DB_URL} ../../../venv/bin/poetry run python apply.py"
  end
end

task :db_new_revision do
  Dir.chdir('server/kraken/migrations') do
    comment = ENV['comment']
    sh "KRAKEN_DB_URL=#{DB_URL} ../../../venv/bin/poetry run alembic revision -m '#{comment}' --autogenerate"
  end
end


# DOCKER Support

task :docker_srv_builder do
  Dir.chdir('server') do
    sh "docker build -f docker-builder.txt -t krakenci/srv-builder:#{kk_stamp} ."
    sh "docker push krakenci/srv-builder:#{kk_stamp}"
  end
end

task :docker_srv_base do
  Dir.chdir('server') do
    sh "docker build -f docker-base.txt -t krakenci/srv-base:#{kk_stamp} ."
    sh "docker push krakenci/srv-base:#{kk_stamp}"
  end
end


# DOCKER

task :docker_up => [DOCKER_COMPOSE, :build_all] do
  sh "cp dot.env .env"
  sh "#{DOCKER_COMPOSE} down"
  sh "#{DOCKER_COMPOSE} build --build-arg kkver=#{kk_ver}"
  sh "#{DOCKER_COMPOSE} up"
end

task :docker_down => DOCKER_COMPOSE do
  sh "#{DOCKER_COMPOSE} down -v --remove-orphans"
end

task :docker_test_up => [DOCKER_COMPOSE] do
  sh "cp dot.env .env"
  sh "cp docker-compose.yaml docker-compose-test.yaml"
  sh "sed -i -e s/kk_ver/#{kk_ver}/g docker-compose-test.yaml"
  sh "sed -i -e 's#127.0.0.1:5000#us-docker.pkg.dev/kraken-261806/kk#g' docker-compose-test.yaml"
  sh "yq e 'del(.services.*.build)' -i docker-compose-test.yaml"
  sh "yq e 'del(.services.agent.deploy)' -i docker-compose-test.yaml"
  sh "#{DOCKER_COMPOSE} -f docker-compose-test.yaml down -v"
  sh "#{DOCKER_COMPOSE} -f docker-compose-test.yaml up"
end

task :run_ch => DOCKER_COMPOSE do
  sh "#{DOCKER_COMPOSE} up clickhouse clickhouse-proxy"
end

task :run_minio => DOCKER_COMPOSE do
  sh "#{DOCKER_COMPOSE} up minio"
end


task :build_chp => DOCKER_COMPOSE do
  sh "#{DOCKER_COMPOSE} build --build-arg kkver=#{kk_ver} clickhouse-proxy"

  dkr_id = `docker create 127.0.0.1:5000/kkchproxy:kk_ver`
  dkr_id = dkr_id.strip
  sh "docker cp #{dkr_id}:/proxy/clickhouse-proxy - | tar -C clickhouse-proxy -xvf -"
  sh "docker rm -v #{dkr_id}"
end

task :build_ch => DOCKER_COMPOSE do
  sh "#{DOCKER_COMPOSE} build --build-arg kkver=#{kk_ver} clickhouse"
end

task :run_chcli do
  sh 'docker run -it --rm --network kraken_db_net --link kraken_clickhouse_1:clickhouse-server yandex/clickhouse-client --host clickhouse-server'
end

task :run_pgsql do
  sh 'docker run --rm -p 5433:5432 -e POSTGRES_USER=kraken -e POSTGRES_PASSWORD=kk123 -e POSTGRES_DB=kraken postgres:11'
end

task :run_redis do
  sh 'docker run --rm -p 6379:6379 redis:alpine'
end

task :build_docker_deploy => DOCKER_COMPOSE do
  sh "#{DOCKER_COMPOSE} -f docker-compose-swarm.yaml config > docker-compose-swarm-deploy.yaml"
end

task :build_docker => DOCKER_COMPOSE do
  sh 'cp dot.env .env'

  # generate docker-compose config for installing kraken under the desk and for pushing images to docker images repository
  sh "cp docker-compose.yaml kraken-docker-compose-#{kk_ver}-tmp.yaml"
  sh "sed -i -e s/kk_ver/#{kk_ver}/g kraken-docker-compose-#{kk_ver}-tmp.yaml"
  sh "sed -i -e 's#127.0.0.1:5000#us-docker.pkg.dev/kraken-261806/kk#g' kraken-docker-compose-#{kk_ver}-tmp.yaml"
  sh "cp agent/kkagent agent/kktool server/"
  sh 'cp README.md server/'

  # build images, in case of server everything is build in containers
  if ENV['reuse'] == 'true'
    flags = ''
  else
    flags = '--force-rm --no-cache --pull'
  end
  sh "#{DOCKER_COMPOSE} -f kraken-docker-compose-#{kk_ver}-tmp.yaml build #{flags} --build-arg kkver=#{kk_ver}"
  sh 'rm server/README.md'

  # extract clickhouse-proxy
  chp_img = "us-docker.pkg.dev/kraken-261806/kk/kkchproxy:#{kk_ver}"
  dkr_id = `docker create #{chp_img}`
  dkr_id = dkr_id.strip
  sh "docker cp #{dkr_id}:/proxy/clickhouse-proxy - | gzip > #{KK_CH_PROXY_TGZ}"
  sh "docker rm -v #{dkr_id}"
end

task :publish_docker => DOCKER_COMPOSE do
  Rake::Task["build_docker"].invoke

  # push built images
  sh "#{DOCKER_COMPOSE} -f kraken-docker-compose-#{kk_ver}-tmp.yaml push"
  # strip build: section - release docker-compose file should use build images directly
  sh "yq e 'del(.services.*.build)' -i kraken-docker-compose-#{kk_ver}-tmp.yaml"
  # strip deploy: section - deploy is used only in swarm
  sh "yq e 'del(.services.*.deploy)' -i kraken-docker-compose-#{kk_ver}-tmp.yaml"
  # validate final docker compose file
  sh "#{DOCKER_COMPOSE} -f kraken-docker-compose-#{kk_ver}-tmp.yaml config > /dev/null"
  sh "mv kraken-docker-compose-#{kk_ver}-tmp.yaml kraken-docker-compose-#{kk_ver}.yaml"
end

task :publish_docker_aws do
  names = ['kkserver:',
           'kkcontroller:',
           'kkrq:',
           'kkagent:',
           'kkui:',
           'clickhouse-server:22.10.2.11.',
           'kkchproxy:']
  names.each do |name|
    sh "docker tag us-docker.pkg.dev/kraken-261806/kk/#{name}#{kk_ver} public.ecr.aws/kraken-ci/#{name}#{kk_ver}"
    sh "docker push public.ecr.aws/kraken-ci/#{name}#{kk_ver}"
  end
end

task :prune_gcp_artifacts do
  ago = (Date.today - 20)
  sh "gcloud artifacts docker images list us-docker.pkg.dev/kraken-261806/kk --include-tags --filter='tags!~released AND createTime<=#{ago}' --format='value[separator=\"@\"](package,version)' | xargs -n1 gcloud artifacts docker images delete --quiet --delete-tags --async"
end

task :show_old_gcp_artifacts do
  ago = (Date.today - 20)
  sh "gcloud artifacts docker images list us-docker.pkg.dev/kraken-261806/kk --include-tags --filter='tags!~released AND createTime<=#{ago}'"
end

task :mark_images_as_published do
  names = ['kkserver:',
           'kkcontroller:',
           'kkrq:',
           'kkagent:',
           'kkui:',
           'clickhouse-server:22.10.2.11.',
           'kkchproxy:']
  names.each do |name|
    sh "gcloud artifacts docker tags add us-docker.pkg.dev/kraken-261806/kk/#{name}#{kk_ver} us-docker.pkg.dev/kraken-261806/kk/#{name}#{kk_ver}-released"
  end
end


task :compose_to_swarm => DOCKER_COMPOSE do
  sh 'cp lab.env .env'
  sh 'cp docker-compose.yaml docker-compose-swarm-tmp.yaml'
  sh "yq e 'del(.services.*.depends_on)'                         -i docker-compose-swarm-tmp.yaml"
  sh "yq e 'del(.services.*.build)'                              -i docker-compose-swarm-tmp.yaml"
#  sh "yq e 'del(.services.*.networks)'                           -i docker-compose-swarm-tmp.yaml"
#  sh "yq e 'del(.networks)'                                      -i docker-compose-swarm-tmp.yaml"
  sh "yq e '.services.ui.ports = [\"8888:80\"]'                  -i docker-compose-swarm-tmp.yaml"
  sh "yq e 'del(.services.clickhouse.ports)'                     -i docker-compose-swarm-tmp.yaml"
  sh "yq e '.services.agent.environment = .services.agent.environment + \"KRAKEN_AGENT_SLOT={{.Task.Slot}}\"' -i docker-compose-swarm-tmp.yaml"
  sh "yq eval-all 'select(fileIndex == 0) * select(filename == \"docker-compose-swarm-patch.yaml\")' docker-compose-swarm-tmp.yaml docker-compose-swarm-patch.yaml > docker-compose-swarm.yaml"
  sh 'rm docker-compose-swarm-tmp.yaml'
  sh "#{DOCKER_COMPOSE} -f docker-compose-swarm.yaml config > kraken-docker-stack-#{kk_ver}.yaml"
  sh 'rm docker-compose-swarm.yaml'
  sh "sed -i -e s/kk_ver/#{kk_ver}/g kraken-docker-stack-#{kk_ver}.yaml"
  sh "yq e '.version = \"3.6\"'                                  -i kraken-docker-stack-#{kk_ver}.yaml"
end

task :run_swarm => :build_docker_deploy do
  sh 'docker stack deploy --with-registry-auth -c docker-compose-swarm-deploy.yaml kraken'
end

task :run_portainer do
  sh 'docker volume create portainer_data'
  sh 'docker run -d -p 8000:8000 -p 9000:9000 -v /var/run/docker.sock:/var/run/docker.sock -v portainer_data:/data portainer/portainer'
end


# system tests

file PULUMI do
  sh "mkdir -p #{TOOLS_DIR}"
  Dir.chdir(TOOLS_DIR) do
    sh "wget -nv https://github.com/pulumi/pulumi/releases/download/#{PULUMI_VER}/pulumi-#{PULUMI_VER}-linux-x64.tar.gz"
    sh "tar xzf pulumi-#{PULUMI_VER}-linux-x64.tar.gz"
    sh "mv pulumi pulumi-#{PULUMI_VER}"
    sh "rm -rf pulumi-#{PULUMI_VER}-linux-x64.tar.gz"
  end
end

task :pulumi_init => PULUMI do
  Dir.chdir('pulumi/ecs') do
    sh "#{PULUMI} login --local"
    sh "PULUMI_CONFIG_PASSPHRASE= #{PULUMI} stack init aws-ecs --secrets-provider passphrase"
    sh "PULUMI_CONFIG_PASSPHRASE= #{PULUMI} config set aws:region ca-central-1"
    sh "PULUMI_CONFIG_PASSPHRASE= #{PULUMI} config set kk_ver #{kk_ver}"
  end
end

task :run_systests => PULUMI do
  # set empty string as passphrase to pulumi to make it happy
  ENV["PULUMI_CONFIG_PASSPHRASE"] = ""

  # setup Kraken in AWS ECS
  kraken_addr = ''
  Dir.chdir('pulumi/ecs') do
    sh "#{PULUMI} up -f -y || (#{PULUMI} destroy -f -y && false)"
    kraken_addr = `#{PULUMI} stack output 'Kraken Server Address'`
  end
  kraken_addr = kraken_addr.strip

  # TODO: wait until Kraken is ready
  sh 'sleep 120'

  # run system tests
  Dir.chdir('tests') do
    sh "KRAKEN_ADDR=#{kraken_addr} pytest-3 systests.py -s --junit-xml=systests-results.xml || true"
  end

  # teardown Kraken in AWS ECS
  Dir.chdir('pulumi/ecs') do
    sh "#{PULUMI} destroy -f -y"
  end
end


# Helm packaging

file HELM do
  sh "mkdir -p #{TOOLS_DIR}"
  Dir.chdir(TOOLS_DIR) do
    sh "wget -nv https://get.helm.sh/helm-#{HELM_VER}-#{sysnamel}-amd64.tar.gz"
    sh "tar xzf helm-#{HELM_VER}-#{sysnamel}-amd64.tar.gz #{sysnamel}-amd64/helm"
    sh "mv #{sysnamel}-amd64/helm #{HELM}"
    sh "rm -rf #{sysnamel}-amd64"
  end
end

task :helm_pkg => HELM do
  sh "#{HELM} lint ./helm --strict"
  sh "#{HELM} package ./helm --app-version #{kk_ver} --version #{kk_ver} -d #{helm_dest}"
end

task :helm_deploy => :helm_pkg do
  sh "#{HELM} upgrade kk --install --create-namespace --namespace kk --debug --wait --set access.external_ips={`minikube ip`} --set access.method='external-ips' kraken-ci-#{kk_ver}.tgz"
end

task :helm_upload do
  sh "#{HELM} repo index #{helm_dest} --url https://kraken.ci/helm-repo/charts"
  Dir.chdir(helm_dest) do
    sh "git add kraken-ci-#{kk_ver}.tgz"
    sh "git commit -am 'added new kraken version #{kk_ver}'"
    sh "git push"
  end
end

task :helm_release do
  Rake::Task['helm_pkg'].invoke
  Rake::Task['helm_upload'].invoke
end

task :deploy_lab => ['./venv/bin/python3'] do
  # prepare docker stack config
  Rake::Task["compose_to_swarm"].invoke

  # deploy to lab.kraken.ci
  sh "./venv/bin/fab -e -H #{ENV['host']} upgrade --kk-ver #{kk_ver}"
end

task :github_release do
  curl_opts = '--retry 3 --silent --location -H "Authorization: token $GITHUB_TOKEN"'

  # create release entry
  sh "curl #{curl_opts} --fail --data '{\"tag_name\": \"v#{kk_ver}\"}' -o github-release-#{kk_ver}.json  https://api.github.com/repos/kraken-ci/kraken/releases"

  # upload artifacts
  file = File.read("github-release-#{kk_ver}.json")
  rel = JSON.parse(file)
  upload_url = rel['upload_url'].chomp('{?name,label}')
  curl_opts2 = "#{curl_opts} -H 'Content-Type:text/plain'"
  sh "curl #{curl_opts2} --data-binary @kraken-docker-compose-#{kk_ver}.yaml '#{upload_url}?name=kraken-docker-compose-#{kk_ver}.yaml'"
  sh "curl #{curl_opts2} --data-binary @dot.env '#{upload_url}?name=krakenci-#{kk_ver}.env'"
  sh "curl #{curl_opts2} --data-binary @#{KK_SERVER_TGZ_PATH} '#{upload_url}?name=#{KK_SERVER_TGZ}'"
  sh "curl #{curl_opts2} --data-binary @#{KK_CLIENT_TGZ_PATH} '#{upload_url}?name=#{KK_CLIENT_TGZ}'"
  sh "curl #{curl_opts2} --data-binary @#{KK_WEB_UI_TGZ_PATH} '#{upload_url}?name=#{KK_WEB_UI_TGZ}'"
  sh "curl #{curl_opts2} --data-binary @#{KK_AGENT_TGZ_PATH} '#{upload_url}?name=#{KK_AGENT_TGZ}'"
  sh "curl #{curl_opts2} --data-binary @#{KK_CH_PROXY_TGZ} '#{upload_url}?name=#{KK_CH_PROXY_TGZ}'"

  sh "rm -f github-release-#{kk_ver}.json"

  # generate and set release notes
  ENV['GREN_GITHUB_TOKEN'] = ENV['GITHUB_TOKEN']
#  Dir.chdir('ui') do
  sh 'npm install github-release-notes'
#  end
  sh './ui/node_modules/.bin/gren release --override'
end

task :release_deploy do
  Rake::Task["build_all"].invoke
  Rake::Task["publish_docker"].invoke
#  Rake::Task["github_release"].invoke
  Rake::Task["deploy_lab"].invoke
end