diff --git a/ci/tasks/test-integration.sh b/ci/tasks/test-integration.sh index a39518bebf..73c4ec309d 100755 --- a/ci/tasks/test-integration.sh +++ b/ci/tasks/test-integration.sh @@ -8,116 +8,13 @@ source bosh-src/ci/tasks/utils.sh check_param RUBY_VERSION check_param DB -echo "Starting $DB..." -case "$DB" in - mysql) - export DB_USER="root" - export DB_PASSWORD="password" - export DB_PORT="3306" - if [ ! -d /var/lib/mysql-src ]; then # Set up MySQL if it's the first time - mv /var/lib/mysql /var/lib/mysql-src - mkdir -p /var/lib/mysql - mount -t tmpfs -o size=512M tmpfs /var/lib/mysql - mv /var/lib/mysql-src/* /var/lib/mysql/ - fi - echo ' -[client] -default-character-set=utf8 +start_db "${DB}" -[mysql] -default-character-set=utf8 - -[mysqld] -collation-server = utf8_unicode_ci -init-connect='SET NAMES utf8' -character-set-server = utf8 -sql-mode="STRICT_TRANS_TABLES" -skip-log-bin -max_connections = 1024' >> /etc/mysql/my.cnf - - echo "....... DB TLS enabled ......." - - export MYSQLDIR=/var/lib/mysql - cp bosh-src/src/bosh-dev/assets/sandbox/database/database_server/private_key $MYSQLDIR/server-key.pem - cp bosh-src/src/bosh-dev/assets/sandbox/database/database_server/certificate.pem $MYSQLDIR/server-cert.pem - echo ' -ssl-cert=server-cert.pem -ssl-key=server-key.pem -require_secure_transport=ON -max_allowed_packet=6M' >> /etc/mysql/my.cnf - - service mysql start - sleep 5 - mysql -h 127.0.0.1 \ - -P ${DB_PORT} \ - --user=${DB_USER} \ - --password=${DB_PASSWORD} \ - -e 'create database uaa;' > /dev/null 2>&1 - ;; - postgresql) - export PATH=/usr/lib/postgresql/$DB_VERSION/bin:$PATH - export DB_USER="postgres" - export DB_PASSWORD="smurf" - export DB_PORT="5432" - export PGPASSWORD=${DB_PASSWORD} - - if [ ! -d /tmp/postgres ]; then # PostgreSQL hasn't been set up - mkdir /tmp/postgres - mount -t tmpfs -o size=512M tmpfs /tmp/postgres - mkdir /tmp/postgres/data - chown postgres:postgres /tmp/postgres/data - export PGDATA=/tmp/postgres/data - - su -m postgres -c ' - export PATH=/usr/lib/postgresql/$DB_VERSION/bin:$PATH - export PGDATA=/tmp/postgres/data - export PGLOGS=/tmp/log/postgres - mkdir -p $PGDATA - mkdir -p $PGLOGS - echo $DB_PASSWORD > /tmp/bosh-postgres.password - initdb -U postgres -D $PGDATA --pwfile /tmp/bosh-postgres.password - ' - - echo "max_connections = 1024" >> $PGDATA/postgresql.conf - echo "shared_buffers = 240MB" >> $PGDATA/postgresql.conf - - echo "....... DB TLS enabled ......." - cp bosh-src/src/bosh-dev/assets/sandbox/database/database_server/private_key $PGDATA/server.key - cp bosh-src/src/bosh-dev/assets/sandbox/database/database_server/certificate.pem $PGDATA/server.crt - chown postgres $PGDATA/server.key - chown postgres $PGDATA/server.crt - su postgres -c ' - export PGDATA=/tmp/postgres/data - echo "ssl = on" >> $PGDATA/postgresql.conf - echo "client_encoding = 'UTF8'" >> $PGDATA/postgresql.conf - echo "hostssl all all 127.0.0.1/32 password" > $PGDATA/pg_hba.conf - echo "hostssl all all 0.0.0.0/32 password" >> $PGDATA/pg_hba.conf - echo "hostssl all all ::1/128 password" >> $PGDATA/pg_hba.conf - echo "hostssl all all localhost password" >> $PGDATA/pg_hba.conf - - chmod 600 $PGDATA/server.* - ' - - su postgres -c ' - export PATH=/usr/lib/postgresql/$DB_VERSION/bin:$PATH - export PGLOGS=/tmp/log/postgres - export PGCLIENTENCODING=UTF8 - pg_ctl start -l $PGLOGS/server.log -o "-N 400" --wait - createdb -h 127.0.0.1 uaa - ' - fi - ;; - *) - echo "Usage: DB={mysql|postgresql} $0 {commands}" - exit 1 -esac - -cp ./bosh-cli/*bosh-cli-*-linux-amd64 /usr/local/bin/bosh -chmod +x /usr/local/bin/bosh +install ./bosh-cli/*bosh-cli-*-linux-amd64 /usr/local/bin/bosh agent_path=bosh-src/src/go/src/github.com/cloudfoundry/ -mkdir -p $agent_path -cp -r bosh-agent $agent_path +mkdir -p "${agent_path}" +cp -r bosh-agent "${agent_path}" pushd bosh-src/src print_git_state @@ -129,10 +26,6 @@ pushd bosh-src/src bundle exec rake --trace spec:integration bundle_exit_code=$? - - if [[ "$DB" = "mysql" ]]; then - service mysql stop - fi popd mkdir -p parallel-runtime-log diff --git a/ci/tasks/test-unit.sh b/ci/tasks/test-unit.sh index b2606c43fc..8174e9f65f 100755 --- a/ci/tasks/test-unit.sh +++ b/ci/tasks/test-unit.sh @@ -5,49 +5,12 @@ set -e source bosh-src/ci/tasks/utils.sh check_param RUBY_VERSION -echo "Starting $DB..." -case "$DB" in - mysql) - mv /var/lib/mysql /var/lib/mysql-src - mkdir /var/lib/mysql - mount -t tmpfs -o size=512M tmpfs /var/lib/mysql - mv /var/lib/mysql-src/* /var/lib/mysql/ +start_db "${DB}" - service mysql start - ;; - postgresql) - mkdir /tmp/postgres - mount -t tmpfs -o size=512M tmpfs /tmp/postgres - mkdir /tmp/postgres/data - chown postgres:postgres /tmp/postgres/data +pushd bosh-src/src + print_git_state - # shellcheck disable=SC2016 - su -m postgres -c ' - export PATH=/usr/lib/postgresql/$DB_VERSION/bin:$PATH - export PGDATA=/tmp/postgres/data - export PGLOGS=/tmp/log/postgres - mkdir -p $PGLOGS - initdb -U postgres -D $PGDATA - - echo "checkpoint_timeout=1h" >> $PGDATA/postgresql.conf - echo "min_wal_size=300MB" >> $PGDATA/postgresql.conf - echo "max_wal_size=300MB" >> $PGDATA/postgresql.conf - - pg_lsclusters - pg_ctlcluster $DB_VERSION main start - ' - ;; - sqlite) - echo "Using sqlite" - ;; - *) - echo "Usage: DB={mysql2|postgresql|sqlite} $0 {commands}" - exit 1 -esac - -cd bosh-src/src -print_git_state - -gem install -f bundler -bundle install --local -bundle exec rake --trace spec:unit + gem install -f bundler + bundle install --local + bundle exec rake --trace spec:unit +popd diff --git a/ci/tasks/utils.sh b/ci/tasks/utils.sh index bf9db6e657..98e091c6f9 100755 --- a/ci/tasks/utils.sh +++ b/ci/tasks/utils.sh @@ -74,13 +74,139 @@ retry_command() { $retryable_command local status=$? if [ $status -ne 0 ]; then - echo "sleeping 3s" + echo "attempt '${i}' exited with '${status}' sleeping 3s" sleep 3s else return 0 fi done set -e - echo "Timed out running command '$retryable_command'" + echo "Timed out running command '${retryable_command}'" return 1 } + +run_as() { + local user="${1}" + shift + + echo "Running '${*}' as '${user}'" + + # shellcheck disable=SC2068 + sudo --preserve-env -u "${user}" ${@} +} + +start_db() { + local db_name=$1 + + echo "Starting ${db_name}..." + + case "${db_name}" in + mysql) + export DB_USER="root" + export DB_PASSWORD="password" + export DB_PORT="3306" + + export MYSQL_ROOT=/var/lib/mysql + if [ ! -d /var/lib/mysql-src ]; then # Set up MySQL if it's the first time + mv "${MYSQL_ROOT}" /var/lib/mysql-src + mkdir -p "${MYSQL_ROOT}" + mount -t tmpfs -o size=512M tmpfs "${MYSQL_ROOT}" + mv /var/lib/mysql-src/* "${MYSQL_ROOT}/" + + echo "Copy 'src/bosh-dev/assets/sandbox/database/database_server/{private_key,certificate.pem}' to '${MYSQL_ROOT}/'" + cp bosh-src/src/bosh-dev/assets/sandbox/database/database_server/private_key "${MYSQL_ROOT}/server.key" + cp bosh-src/src/bosh-dev/assets/sandbox/database/database_server/certificate.pem "${MYSQL_ROOT}/server.cert" + + { + echo "[client]" + echo "default-character-set=utf8" + echo "[mysql]" + echo "default-character-set=utf8" + + echo "[mysqld]" + echo "collation-server = utf8_unicode_ci" + echo "init-connect='SET NAMES utf8'" + echo "character-set-server = utf8" + echo 'sql-mode="STRICT_TRANS_TABLES"' + echo "skip-log-bin" + echo "max_connections = 1024" + + echo "ssl-cert=server.cert" + echo "ssl-key=server.key" + echo "require_secure_transport=ON" + echo "max_allowed_packet=6M" + } >> /etc/mysql/my.cnf + fi + + service mysql start + sleep 5 + mysql -h 127.0.0.1 \ + -P ${DB_PORT} \ + --user=${DB_USER} \ + --password=${DB_PASSWORD} \ + -e 'create database uaa;' > /dev/null 2>&1 + ;; + + postgresql) + export DB_USER="postgres" + export DB_PASSWORD="smurf" + export DB_PORT="5432" + export PGPASSWORD="${DB_PASSWORD}" + + export POSTGRES_ROOT="/tmp/postgres" + if [ ! -d "${POSTGRES_ROOT}" ]; then # PostgreSQL hasn't been set up + mkdir -p "${POSTGRES_ROOT}" + mount -t tmpfs -o size=512M tmpfs "${POSTGRES_ROOT}" + + export PGDATA="${POSTGRES_ROOT}/data" + export PGLOGS="/tmp/log/postgres" + export PGCLIENTENCODING="UTF8" + + run_as postgres mkdir -p "${PGDATA}" "${PGLOGS}" + + export POSTGRES_PASSWORD_FILE="${POSTGRES_ROOT}/bosh-postgres.password" + echo "${DB_PASSWORD}" > "${POSTGRES_PASSWORD_FILE}" + chown -R postgres:postgres "${PGDATA}" + run_as postgres "$(which initdb)" -U postgres -D "${PGDATA}" --pwfile "${POSTGRES_PASSWORD_FILE}" + + # NOTE: certificates can only moved to ${PGDATA}/ _after_ `initdb` is run + echo "Copy 'src/bosh-dev/assets/sandbox/database/database_server/{private_key,certificate.pem}' to '${PGDATA}'" + cp bosh-src/src/bosh-dev/assets/sandbox/database/database_server/private_key "${PGDATA}/server.key" + cp bosh-src/src/bosh-dev/assets/sandbox/database/database_server/certificate.pem "${PGDATA}/server.crt" + chmod 600 ${PGDATA}/server.* + + export POSTGRES_CONF="${PGDATA}/postgresql.conf" + export POSTGRES_PG_HBA="${PGDATA}/pg_hba.conf" + + { + echo "max_connections = 1024" + echo "shared_buffers = 240MB" + echo "ssl = on" + echo "client_encoding = 'UTF8'" + } >> "${POSTGRES_CONF}" + + echo "hostssl all all 127.0.0.1/32 password" > "${POSTGRES_PG_HBA}" + { + echo "hostssl all all 0.0.0.0/32 password" + echo "hostssl all all ::1/128 password" + echo "hostssl all all localhost password" + } >> "${POSTGRES_PG_HBA}" + + chown -R postgres:postgres "${PGDATA}" "${PGLOGS}" + + run_as postgres "$(which pg_ctl)" start --log="${PGLOGS}/server.log" --options='-N 400' --wait + run_as postgres "$(which createdb)" -h 127.0.0.1 uaa + fi + ;; + + sqlite) + echo "Using sqlite" + echo " NOTE: this will not work for integration specs" + ;; + + *) + + echo "Usage: DB={mysql|postgresql|sqlite} $0 {commands}" + exit 1 + esac +} diff --git a/src/bosh-dev/lib/bosh/dev/tasks/fly.rake b/src/bosh-dev/lib/bosh/dev/tasks/fly.rake index 8153e38a8c..5be19fe598 100644 --- a/src/bosh-dev/lib/bosh/dev/tasks/fly.rake +++ b/src/bosh-dev/lib/bosh/dev/tasks/fly.rake @@ -4,7 +4,7 @@ namespace :fly do # bundle exec rake fly:unit desc 'Fly unit specs' task :unit do - db, db_version = fetch_db_and_version + db, db_version = fetch_db_and_version('sqlite') execute('test-unit', command_opts('unit', db, db_version), DB: db, DB_VERSION: db_version, @@ -14,7 +14,7 @@ namespace :fly do # bundle exec rake fly:integration desc 'Fly integration specs' task :integration, [:cli_dir] do |_, args| - db, db_version = fetch_db_and_version + db, db_version = fetch_db_and_version('postgresql') command_opts = command_opts('integration', db, db_version) command_opts += " --input bosh-cli=#{args[:cli_dir]}" if args[:cli_dir] @@ -32,8 +32,8 @@ namespace :fly do private - def fetch_db_and_version - db = ENV.fetch('DB', 'postgresql') + def fetch_db_and_version(default_db) + db = ENV.fetch('DB', default_db) case db when 'postgresql' diff --git a/src/bosh-director/spec/spec_helper.rb b/src/bosh-director/spec/spec_helper.rb index 9c71d44e4a..4770a734b0 100644 --- a/src/bosh-director/spec/spec_helper.rb +++ b/src/bosh-director/spec/spec_helper.rb @@ -97,6 +97,7 @@ def init_database username: ENV['DB_USER'], password: ENV['DB_PASSWORD'], host: ENV['DB_HOST'] || '127.0.0.1', + port: ENV['DB_PORT'], }.compact) else uri = URI.parse(connection_string) @@ -121,7 +122,7 @@ def init_database ) when 'mysql' require File.expand_path('../../bosh-dev/lib/bosh/dev/sandbox/mysql', File.dirname(__FILE__)) - db_options[:port] = 3306 + db_options[:port] ||= 3306 @director_db_helper = Bosh::Dev::Sandbox::Mysql.new( "#{@db_name}_director",