Improve build system; add e2e tests, docker CD, FAKE, paket

.build/docker-ops

@@ -0,0 +1,147 @@
+#!/bin/bash
+
+################################################################################
+# Generally:
+# * Each log output goes to stderrr, so that we can return value from functions
+# with stdout. (Curl does the same).
+# * Do not set here: `set -e`, because after this script is sourced, then
+# using it interactively would be troublesome: exit on each failure and
+# bash completion results in exit too.
+################################################################################
+
+program_name() {
+  echo "DOCKER-OPS"
+}
+log_debug() {
+  if [ "${RELEASER_LOG_LEVEL}" == "debug" ]; then
+    echo -e "$(date "+%d-%m-%Y %T") $(program_name) debug: ${1}" >&2
+  fi
+}
+log_info() {
+  echo -e "$(date "+%d-%m-%Y %T") $(program_name) info: ${1}" >&2
+}
+log_error() {
+  echo -e "\e[31m$(date "+%d-%m-%Y %T") $(program_name) error: $1\e[0m" >&2
+  echo -e "\e[31m$(date "+%d-%m-%Y %T") $(program_name) error: File: ${BASH_SOURCE[0]}\e[0m" >&2
+  echo -e "\e[31m$(date "+%d-%m-%Y %T") $(program_name) error: Line numbers: ${BASH_LINENO[*]}\e[0m" >&2
+}
+log_warn() {
+  echo -e "\e[33m$(date "+%d-%m-%Y %T") $(program_name) warn: $1\e[0m" >&2
+}
+
+# In order to load all the docker-ops functions once you can run:
+# docker_ops || eval "$(curl http://archive.ai-traders.com/docker-ops/0.1.3/docker-ops)"
+# Or you can simply wget that file and source it.
+function docker_ops_loaded {
+  return 0
+}
+
+function docker_build {
+  image_dir="$1"
+  imagerc_filename="$2"
+  image_name="$3"
+  image_tag="$4"
+
+  if [[ -z "${image_dir}" ]];then
+    log_error "image_dir not set"
+    return 1
+  fi
+  log_info "image_dir set to: ${image_dir}"
+  if [[ -z "${image_name}" ]];then
+    log_error "image_name not set"
+    return 1
+  fi
+  log_info "image_name set to: ${image_name}"
+  if [[ -z "${imagerc_filename}" ]];then
+    log_error "imagerc_filename not set"
+    return 1
+  fi
+  log_info "imagerc_filename set to: ${imagerc_filename}"
+  initial_dir="$(pwd)"
+  cd "${image_dir}"
+  if [[ -z "${image_tag}" ]];then
+    # set to the latest git commit hash
+    image_tag=$(git rev-parse HEAD)
+  fi
+  log_info "image_tag set to: ${image_tag}"
+
+  set -x -e
+  docker build ${docker_build_options} -t "${image_name}:${image_tag}" .
+  echo "export AIT_DOCKER_IMAGE_NAME=\"${image_name}\"" > "${imagerc_filename}"
+  echo "export AIT_DOCKER_IMAGE_TAG=\"${image_tag}\"" >> "${imagerc_filename}"
+
+  echo "{ \"docker_image_name\":\"${image_name}\"," > "${imagerc_filename}.json"
+  echo "\"docker_image_tag\":\"${image_tag}\"," >> "${imagerc_filename}.json"
+  echo "\"docker_image_url\":\"${image_name}:${image_tag}\" }" >> "${imagerc_filename}.json"
+
+  echo "---" > "${imagerc_filename}.yml"
+  echo "docker_image_name: ${image_name}" >> "${imagerc_filename}.yml"
+  echo "docker_image_tag: ${image_tag}" >> "${imagerc_filename}.yml"
+  echo "docker_image_url: ${image_name}:${image_tag}" >> "${imagerc_filename}.yml"
+  if [[ "${dryrun}" != "true" && "${ops_docker_push}" != "false" ]];then
+    docker push "${image_name}:${image_tag}"
+  fi
+  cd "${initial_dir}"
+  set +x +e
+}
+function source_imagerc {
+  image_dir="$1"
+  imagerc_filename="$2"
+
+  if [[ -z "${image_dir}" ]];then
+    log_error "image_dir not set"
+    return 1
+  fi
+  if [[ -z "${imagerc_filename}" ]];then
+    log_error "imagerc_filename not set"
+    return 1
+  fi
+  set -x -e
+  source "${image_dir}/${imagerc_filename}"
+  set +x +e
+}
+function docker_push {
+  image_name="$1"
+  testing_image_tag="$2"
+  production_image_tag="$3"
+
+  if [[ -z "${image_name}" ]];then
+    log_error "image_name not set"
+    return 1
+  fi
+  log_info "image_name set to: ${image_name}"
+  if [[ -z "${testing_image_tag}" ]];then
+    log_error "testing_image_tag not set"
+    return 1
+  fi
+  log_info "testing_image_tag set to: ${testing_image_tag}"
+  if [[ -z "${production_image_tag}" ]];then
+    log_error "production_image_tag not set"
+    return 1
+  fi
+  log_info "production_image_tag set to: ${production_image_tag}"
+  if ! docker images ${image_name} | awk '{print $2}' | grep ${testing_image_tag} 1>/dev/null ; then
+    # if docker image does not exist locally, then "docker tag" will fail,
+    # so pull it. However, do not always pull it, the image may be not pushed
+    # and only available locally.
+    set -x -e
+    docker pull "${image_name}:${testing_image_tag}"
+  fi
+  set -x -e
+  # When tagging a docker image using docker 1.8.3, we can use `docker tag -f`.
+  # When using docker 1.12, there is no `-f` option, but `docker tag`
+  # always works as if force was used.
+  docker tag -f "${image_name}:${testing_image_tag}" "${image_name}:${production_image_tag}" || docker tag "${image_name}:${testing_image_tag}" "${image_name}:${production_image_tag}"
+  docker tag -f "${image_name}:${testing_image_tag}" "${image_name}:latest" || docker tag "${image_name}:${testing_image_tag}" "${image_name}:latest"
+  if [[ "${dryrun}" != "true" ]];then
+    docker push "${image_name}:${production_image_tag}"
+    docker push "${image_name}:latest"
+  fi
+  set +x +e
+}
+
+### Defaults
+# allow those variables to be already set, so that user can set them in cli
+imagerc_filename="${imagerc_filename:-imagerc}"
+docker_build_options="${docker_build_options:-}"
+ops_docker_push="${ops_docker_push:-false}"

.build/releaser

@@ -0,0 +1,205 @@
+#!/bin/bash
+
+################################################################################
+# Generally:
+# * Each log output goes to stderrr, so that we can return value from functions
+# with stdout. (Curl does the same).
+# * Do not set here: `set -e`, because after this script is sourced, then
+# using it interactively would be troublesome: exit on each failure and
+# bash completion results in exit too.
+################################################################################
+# The Helpers functions:
+#  * are independent of the end user functions and all the variables sourced
+# from releaserrc
+#  * do not assume any default arguments, all variables must be explicitly set.
+#  * are here to have higher test coverage.
+################################################################################
+
+program_name() {
+  echo "RELEASER"
+}
+log_debug() {
+  if [ "${RELEASER_LOG_LEVEL}" == "debug" ]; then
+    echo -e "$(LANG=en_US date "+%d-%m-%Y %T") $(program_name) debug: ${1}" >&2
+  fi
+}
+log_info() {
+  echo -e "$(LANG=en_US date "+%d-%m-%Y %T") $(program_name) info: ${1}" >&2
+}
+log_error() {
+  echo -e "\e[31m$(LANG=en_US date "+%d-%m-%Y %T") $(program_name) error: $1\e[0m" >&2
+  echo -e "\e[31m$(LANG=en_US date "+%d-%m-%Y %T") $(program_name) error: File: ${BASH_SOURCE[0]}\e[0m" >&2
+  echo -e "\e[31m$(LANG=en_US date "+%d-%m-%Y %T") $(program_name) error: Line numbers: ${BASH_LINENO[*]}\e[0m" >&2
+}
+log_warn() {
+  echo -e "\e[33m$(LANG=en_US date "+%d-%m-%Y %T") $(program_name) warn: $1\e[0m" >&2
+}
+
+# In order to load all the releaser functions once you can run:
+# releaser_loaded || eval "$(curl http://gogs.ai-traders.com/platform/releaser/raw/0.2.0/src/releaser)"
+# Or you can simply wget that file and source it.
+function releaser_loaded {
+  return 0
+}
+
+# Arguments:
+#   changelog_file
+function get_last_version_from_changelog {
+  changelog_file="$1"
+  if [[ -z "${changelog_file}" ]];then
+    log_error "changelog_file not set"
+    return 1
+  fi
+  if [[ ! -f "${changelog_file}" ]];then
+    log_error "${changelog_file} does not exist"
+    return 1
+  fi
+  version=$(cat $changelog_file | head -1 | grep -o -e "[0-9]*\.[0-9]*\.[0-9]*")
+  if [[ "${version}" == "" ]];then
+    log_error "Could not get last version from ${changelog_file}"
+    return 1
+  fi
+  echo "${version}"
+}
+# Arguments: none
+function get_last_git_tagged_version {
+  tag="$(git tag 2>&1 | sort -V | tail -1)"
+  if [[ "${tag}" == *"Not a git repository"* ]];then
+    log_error "Not a git repository"
+    return 1
+  fi
+  echo "${tag}"
+}
+
+# Arguments:
+#   * version
+function validate_version_is_semver {
+  version="$1"
+  if [[ -z "${version}" ]];then
+    log_error "version not set"
+    return 1
+  fi
+  if [[ ! "${version}" =~ ^[0-9]+\.[0-9]+\.[0-9]+ ]]; then
+    log_error "Version was not SemVer. It was: ${version}. Must be e.g. 0.1.2"
+    return 1
+  fi
+}
+
+# Replaces the whole line in a version file with that line and adds new version.
+# Arguments:
+#   line_pattern
+#   new_version
+#   version_file
+function set_version_in_file {
+  line_pattern="$1"
+  version_file="$2"
+  new_version="$3"
+
+  if [[ -z "${line_pattern}" ]];then
+    log_error "line_pattern not set"
+    return 1
+  fi
+  if [[ -z "${new_version}" ]];then
+    log_error "new_version not set"
+    return 1
+  fi
+  if [[ -z "${version_file}" ]];then
+    log_error "version_file not set"
+    return 1
+  fi
+  if [[ ! -f "${version_file}" ]];then
+    log_error "${version_file} does not exist"
+    return 1
+  fi
+  new_versionfile_contents=$(sed "s/${line_pattern}.*/${line_pattern}\"${new_version}\"/g" ${version_file})
+  log_debug "Will replace version file with:"
+  log_debug "${new_versionfile_contents}"
+  if [[ "${dryrun}" != "true" ]];then
+    echo "${new_versionfile_contents}" > "${version_file}"
+  fi
+}
+
+# Takes care of the first line of changelog file.
+# There are various cases supported. Suppose we bump to 0.2.0:
+#   Case 1 - changelog 1st line contains older version header or no header
+#     ### 0.1.0 (2017-Apr-29)
+#     Becomes:
+#     ### 0.2.0 (2017-Apr-30)
+#     ### 0.1.0 (2017-Apr-29)
+#   Case 2 - changelog 1st line contains the same version header
+#     ### 0.2.0 (2017-Apr-30)
+#     Becomes: (no changes, even if date is different)
+#     ### 0.2.0 (2017-Apr-30)
+#   Case 3 - changelog 1st line contains Unreleased and the same version header:
+#     ### 0.2.0 - Unreleased
+#     Becomes:
+#     ### 0.2.0 (2017-Apr-29)
+#   Case 4 - changelog 1st line contains Unreleased and older version header:
+#     ### 0.1.0 - Unreleased
+#     Becomes:
+#     ### 0.2.0 (2017-Apr-29)
+#
+# Arguments:
+#   new_version
+#   changelog_file
+function set_version_in_changelog {
+  changelog_file="$1"
+  new_version="$2"
+
+  if [[ -z "${new_version}" ]];then
+    log_error "new_version not set"
+    return 1
+  fi
+  if [[ -z "${changelog_file}" ]];then
+    log_error "changelog_file not set"
+    return 1
+  fi
+  if [[ ! -f "${changelog_file}" ]];then
+    log_error "${changelog_file} does not exist"
+    return 1
+  fi
+
+  changelog_first_line=$(cat ${changelog_file} | head -1)
+  if [[ "${changelog_first_line}" == "### "*" - Unreleased" ]];then
+    log_debug "Setting data in changelog from Unreleased"
+    release_date=$(LANG=en_US date +%Y-%b-%d)
+    new_line="### ${new_version} (${release_date})"
+    new_changelog_contents=$(sed "s/### .* - Unreleased/${new_line}/g" ${changelog_file})
+    if [[ "${dryrun}" != "true" ]];then
+      echo "${new_changelog_contents}" > "${changelog_file}"
+    fi
+    return $?
+  elif [[ "${changelog_first_line}" == "### ${new_version} "* ]];then
+    # idempotent
+    log_debug "Version in changelog is already set"
+    return 0
+  else
+    log_debug "Will add new line to changelog:"
+    release_date=$(LANG=en_US date +%Y-%b-%d)
+    new_line="### ${new_version} (${release_date})"
+    log_debug "${new_line}"
+    if [[ "${dryrun}" != "true" ]];then
+      old_changelog=$(cat "${changelog_file}")
+      echo "${new_line}" > "${changelog_file}"
+      echo "" >> "${changelog_file}"
+      echo "${old_changelog}" >> "${changelog_file}"
+      # this does not work, newline is not printed, even with echo -e:
+      # new_changelog="${new_line}\n${old_changelog}"
+      return $?
+    fi
+  fi
+}
+
+# You should invoke that function before running any end user functions.
+function releaser_init {
+  default_changelog_file="$(pwd)/CHANGELOG.md"
+  changelog_file="${changelog_file:-${default_changelog_file}}"
+  variables_file="$(pwd)/releaserrc"
+  if [[ -f "${variables_file}" ]]; then
+    source "${variables_file}"
+  else
+    log_debug "File: ${variables_file} does not exist, but it is ok."
+  fi
+  log_info "Current directory: $(pwd)"
+  log_debug "Changelog file: ${changelog_file}"
+}

.dockerignore

@@ -0,0 +1,3 @@
+/*
+!/docker-scripts
+!/src/BaGet/bin/Release/netcoreapp2.1/publish

.gitignore

@@ -1,5 +1,6 @@
 ## Ignore Visual Studio temporary files, build results, and
 ## files generated by popular Visual Studio add-ons.
+.paket/.store/
 
 # User-specific files
 *.suo
@@ -266,4 +267,31 @@ __pycache__/
 *.pyc
 
 # Cake - Uncomment if you are using it
-# tools/
+# tools/
+/.paket/paket
+/.paket/paket.bootstrapper
+/*TestsResults.xml
+/*TestResults.html
+/e2e/.paket/paket.exe
+/e2e/data/db/sqlite.db
+/e2e/input
+/e2e/**/*.nupkg
+/e2e/**/*.dll
+
+/imagerc*
+
+# Ignore client app cache
+**/Baget.UI/.cache
+
+# Ignore client dist files
+**/Baget.UI/dist/
+
+# Ignore database file
+**/baget.db
+/src/BaGet.UI/dist/
+/src/BaGet.UI/.cache/
+
+/src/BaGet/Packages/
+/e2e/.paket/paket.bootstrapper.exe.config
+/e2e/test_update-package/paket-constraint/paket.lock
+/e2e/test_update-package/paket/paket.locky client cache was failing