Merge pull request #25 from github/20200729

2020-07-29 Update of code

.github/workflows/misc_test_docker.yml → .github/workflows/docker_test_1.yml

@@ -1,10 +1,11 @@
-name: misc test
+name: docker_test_1
 on: [push, pull_request]
 jobs:
 
  build:
- name: Misc Test
+ name: Docker Test 1
  runs-on: ubuntu-latest
+
  steps:
 
  - name: Set up Go
@@ -15,6 +16,6 @@ jobs:
  - name: Check out code
  uses: actions/checkout@v2
 
- - name: Run Misc test which requires docker
+ - name: Run tests which requires docker 1
  run: |
- go run test.go -docker=true -shard 25
+ go run test.go -docker=true --follow -shard 10

.github/workflows/cluster_vtctl_web.yml → .github/workflows/docker_test_2.yml

@@ -1,10 +1,11 @@
-name: cluster_vtctl_web
+name: docker test 2
 on: [push, pull_request]
 jobs:
 
  build:
- name: cluster vtctl web
+ name: Docker Test 2
  runs-on: ubuntu-latest
+
  steps:
 
  - name: Set up Go
@@ -15,8 +16,6 @@ jobs:
  - name: Check out code
  uses: actions/checkout@v2
 
- - name: Run vtctl web
+ - name: Run tests which requires docker - 2
  run: |
- # Running web test inside docker
- go run test.go -docker=true -print-log -shard 10
-
+ go run test.go -docker=true --follow -shard 25

.github/workflows/golangci-linter.yml

@@ -0,0 +1,23 @@
+name: golangci-lint
+on: [push,pull_request]
+jobs:
+
+ build:
+ name: Build
+ runs-on: ubuntu-latest
+ steps:
+
+ - name: Set up Go 1.13
+ uses: actions/setup-go@v1
+ with:
+ go-version: 1.13
+ id: go
+
+ - name: Check out code into the Go module directory
+ uses: actions/checkout@v1
+
+ - name: Install golangci-lint
+ run: curl -sfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh| sh -s -- -b $(go env GOPATH)/bin v1.27.0
+
+ - name: Run golangci-lint
+ run: $(go env GOPATH)/bin/golangci-lint run --disable=errcheck --timeout=10m go/...

Makefile

@@ -22,7 +22,7 @@ export REWRITER=go/vt/sqlparser/rewriter.go
 # Since we are not using this Makefile for compilation, limiting parallelism will not increase build time.
 .NOTPARALLEL:
 
-.PHONY: all build build_web install test clean unit_test unit_test_cover unit_test_race integration_test proto proto_banner site_test site_integration_test docker_bootstrap docker_test docker_unit_test java_test reshard_tests e2e_test e2e_test_race minimaltools tools
+.PHONY: all build install test clean unit_test unit_test_cover unit_test_race integration_test proto proto_banner site_test site_integration_test docker_bootstrap docker_test docker_unit_test java_test reshard_tests e2e_test e2e_test_race minimaltools tools web_bootstrap web_build web_start
 
 all: build
 
@@ -37,22 +37,11 @@ ifdef VT_EXTRA_BUILD_FLAGS
 export EXTRA_BUILD_FLAGS := $(VT_EXTRA_BUILD_FLAGS)
 endif
 
-# This target needs to be manually run every time any file within web/vtctld2/app is modified to regenerate rice-box.go
-embed_static: 
- cd go/vt/vtctld
- go run github.com/GeertJohan/go.rice/rice embed-go
- go build .
-
 embed_config:
  cd go/vt/mysqlctl
  go run github.com/GeertJohan/go.rice/rice embed-go
  go build .
 
-build_web:
- echo $$(date): Building web artifacts
- cd web/vtctld2 && ng build -prod
- cp -f web/vtctld2/src/{favicon.ico,plotly-latest.min.js,primeui-ng-all.min.css} web/vtctld2/dist/
-
 build:
 ifndef NOBANNER
  echo $$(date): Building source tree
@@ -287,6 +276,14 @@ docker_lite_testing:
  chmod -R o=g *
  docker build -f docker/lite/Dockerfile.testing -t vitess/lite:testing .
 
+docker_local:
+ chmod -R o=g *
+ docker build -f docker/local/Dockerfile -t vitess/local .
+
+docker_mini:
+ chmod -R o=g *
+ docker build -f docker/mini/Dockerfile -t vitess/mini .
+
 # This rule loads the working copy of the code into a bootstrap image,
 # and then runs the tests inside Docker.
 # Example: $ make docker_test flavor=mariadb
@@ -351,3 +348,19 @@ client_go_gen:
  # Move and cleanup
  mv vitess.io/vitess/go/vt/topo/k8stopo/client go/vt/topo/k8stopo/
  rmdir -p vitess.io/vitess/go/vt/topo/k8stopo/
+
+# Check prerequisites and install dependencies
+web_bootstrap:
+ ./tools/web_bootstrap.sh
+
+# Do a production build of the vtctld UI.
+# This target needs to be manually run every time any file within web/vtctld2/app 
+# is modified to regenerate rice-box.go
+web_build: web_bootstrap
+ ./tools/web_build.sh
+
+# Start a front-end dev server with hot reloading on http://localhost:4200.
+# This expects that you have a vtctld API server running on http://localhost:15000.
+# Following the local Docker install guide is recommended: https://vitess.io/docs/get-started/local-docker/
+web_start: web_bootstrap
+ cd web/vtctld2 && npm run start

build.env

@@ -43,3 +43,4 @@ mkdir -p .git/hooks
 ln -sf "$PWD/misc/git/pre-commit" .git/hooks/pre-commit
 ln -sf "$PWD/misc/git/commit-msg" .git/hooks/commit-msg
 git config core.hooksPath .git/hooks
+export EXTRA_BIN=$PWD/test/bin

config/init_db.sql

@@ -91,7 +91,7 @@ GRANT SELECT, PROCESS, SUPER, REPLICATION CLIENT, RELOAD
 GRANT SELECT, UPDATE, DELETE, DROP
  ON performance_schema.* TO 'vt_monitoring'@'localhost';
 
-# User for Orchestrator (https://github.com/github/orchestrator).
+# User for Orchestrator (https://github.com/openark/orchestrator).
 CREATE USER 'orc_client_user'@'%' IDENTIFIED BY 'orc_client_user_password';
 GRANT SUPER, PROCESS, REPLICATION SLAVE, RELOAD
  ON *.* TO 'orc_client_user'@'%';

config/mycnf/default.cnf

@@ -19,7 +19,6 @@ server-id = {{.ServerID}}
 # additional configuration (like enabling semi-sync) before we connect to
 # the master.
 skip_slave_start
-slave_load_tmpdir = {{.SlaveLoadTmpDir}}
 socket = {{.SocketFile}}
 tmpdir = {{.TmpDir}}
 

config/tablet/default.yaml

@@ -78,16 +78,27 @@ oltp:
  maxRows: 10000 # queryserver-config-max-result-size
  warnRows: 0 # queryserver-config-warn-result-size
 
+healthcheck:
+ intervalSeconds: 20 # health_check_interval
+ degradedThresholdSeconds: 30 # degraded_threshold
+ unhealthyThresholdSeconds: 7200 # unhealthy_threshold
+
+gracePeriods:
+ transactionShutdownSeconds: 0 # transaction_shutdown_grace_period
+ transitionSeconds: 0 # serving_state_grace_period
+
+replicationTracker:
+ mode: disable # enable_replication_reporter
+ heartbeatIntervalMilliseconds: 0 # heartbeat_enable, heartbeat_interval
+
 hotRowProtection:
  mode: disable|dryRun|enable # enable_hot_row_protection, enable_hot_row_protection_dry_run
- # Default value is same as txPool.size.
+ # Recommended value: same as txPool.size.
  maxQueueSize: 20 # hot_row_protection_max_queue_size
  maxGlobalQueueSize: 1000 # hot_row_protection_max_global_queue_size
  maxConcurrency: 5 # hot_row_protection_concurrent_transactions
 
 consolidator: enable|disable|notOnMaster # enable-consolidator, enable-consolidator-replicas
-heartbeatIntervalMilliseconds: 0 # heartbeat_enable, heartbeat_interval
-shutdownGracePeriodSeconds: 0 # transaction_shutdown_grace_period
 passthroughDML: false # queryserver-config-passthrough-dmls
 streamBufferSize: 32768 # queryserver-config-stream-buffer-size
 queryCacheSize: 5000 # queryserver-config-query-cache-size

docker/k8s/orchestrator/Dockerfile

@@ -19,7 +19,7 @@ FROM debian:stretch-slim
 RUN apt-get update && \
  apt-get upgrade -qq && \
  apt-get install wget ca-certificates jq -qq --no-install-recommends && \
- wget https://github.com/github/orchestrator/releases/download/v3.1.1/orchestrator_3.1.0_amd64.deb && \
+ wget https://github.com/openark/orchestrator/releases/download/v3.1.1/orchestrator_3.1.0_amd64.deb && \
  dpkg -i orchestrator_3.1.0_amd64.deb && \
  rm orchestrator_3.1.0_amd64.deb && \
  apt-get purge wget -qq && \

docker/local/Dockerfile

@@ -0,0 +1,30 @@
+FROM vitess/bootstrap:common
+
+RUN apt-get update
+RUN apt-get install -y sudo curl vim jq mysql-server
+
+COPY docker/local/install_local_dependencies.sh /vt/dist/install_local_dependencies.sh
+RUN /vt/dist/install_local_dependencies.sh
+RUN echo "source /vt/local/env.sh" >> /etc/bash.bashrc
+
+# Allows some docker builds to disable CGO
+ARG CGO_ENABLED=0
+
+# Re-copy sources from working tree.
+COPY --chown=vitess:vitess . /vt/src/vitess.io/vitess
+
+# Build and install Vitess in a temporary output directory.
+USER vitess
+
+WORKDIR /vt/src/vitess.io/vitess
+RUN make install PREFIX=/vt/install
+
+ENV VTROOT /vt/src/vitess.io/vitess
+ENV VTDATAROOT /vt/vtdataroot
+ENV PATH $VTROOT/bin:$PATH
+ENV PATH="/var/opt/etcd:${PATH}"
+
+RUN mkdir /vt/local
+COPY examples/local /vt/local
+
+CMD cd /vt/local && ./101_initial_cluster.sh && /bin/bash

docker/local/install_local_dependencies.sh

@@ -0,0 +1,22 @@
+#!/bin/bash
+
+# This is a script that gets run as part of the Dockerfile build
+# to install dependencies for the vitess/mini image.
+#
+# Usage: install_mini_dependencies.sh
+
+set -euo pipefail
+
+# Install etcd
+ETCD_VER=v3.4.9
+DOWNLOAD_URL=https://storage.googleapis.com/etcd
+
+curl -k -L ${DOWNLOAD_URL}/${ETCD_VER}/etcd-${ETCD_VER}-linux-amd64.tar.gz -o /tmp/etcd-${ETCD_VER}-linux-amd64.tar.gz
+mkdir -p /var/opt/etcd
+sudo tar xzvf /tmp/etcd-${ETCD_VER}-linux-amd64.tar.gz -C /var/opt/etcd --strip-components=1
+rm -f /tmp/etcd-${ETCD_VER}-linux-amd64.tar.gz
+
+mkdir -p /var/run/etcd && chown -R vitess:vitess /var/run/etcd
+
+# Clean up files we won't need in the final image.
+rm -rf /var/lib/apt/lists/*

docker/local/run.sh

@@ -0,0 +1,3 @@
+#!/bin/bash
+
+docker run -p 15000:15000 -p 15001:15001 --rm -it vitess/local

docker/mini/Dockerfile

@@ -0,0 +1,61 @@
+# Copyright 2019 The Vitess Authors.
+# 
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+# 
+# http://www.apache.org/licenses/LICENSE-2.0
+# 
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# NOTE: We have to build the Vitess binaries from scratch instead of sharing
+# a base image because Docker Hub dropped the feature we relied upon to
+# ensure images contain the right binaries.
+
+# Use a temporary layer for the build stage.
+FROM vitess/base AS base
+
+FROM vitess/lite
+
+USER root
+
+RUN apt-get update
+RUN apt-get install -y sudo curl vim python3 jq sqlite3
+RUN ln -s /usr/bin/python3 /usr/bin/python
+
+# Install minivitess dependencies
+COPY docker/mini/install_mini_dependencies.sh /vt/dist/install_mini_dependencies.sh
+RUN /vt/dist/install_mini_dependencies.sh
+
+COPY docker/mini/orchestrator-vitess-mini.conf.json /etc/orchestrator.conf.json
+RUN chown vitess:vitess /etc/orchestrator.conf.json
+
+COPY docker/mini/docker-entry /vt/dist/docker/mini/docker-entry
+COPY examples/local/scripts /vt/dist/scripts
+COPY examples/local/env.sh /vt/dist/scripts/env.sh
+COPY docker/mini/vtctld-mini-up.sh /vt/dist/scripts/vtctld-mini-up.sh
+COPY docker/mini/vttablet-mini-up.sh /vt/dist/scripts/vttablet-mini-up.sh
+COPY docker/mini/orchestrator-up.sh /vt/dist/scripts/orchestrator-up.sh
+RUN echo "hostname=127.0.0.1" >> /vt/dist/scripts/env.sh
+RUN cat /vt/dist/scripts/env.sh | egrep "^alias" >> /etc/bash.bashrc
+
+COPY --from=base /vt/bin/vtctl /vt/bin/
+COPY --from=base /vt/bin/mysqlctl /vt/bin/
+
+# Set up Vitess environment (just enough to run pre-built Go binaries)
+ENV VTROOT /vt/src/vitess.io/vitess
+ENV VTDATAROOT /vt/vtdataroot
+ENV PATH $VTROOT/bin:$PATH
+ENV PATH="/vt/bin:${PATH}"
+ENV PATH="/var/opt/etcd:${PATH}"
+ENV TOPO="etcd"
+
+# Create mount point for actual data (e.g. MySQL data dir)
+VOLUME /vt/vtdataroot
+USER vitess
+EXPOSE 15000-15200 16000-16200
+CMD /vt/dist/docker/mini/docker-entry && /bin/bash