Merge pull request #12 from SoftwareForScience/feature/jenkins-with-db

Feature/jenkins with db

ci/README.md

@@ -1,5 +1,7 @@
 # Continuous Integration (CI) with Jenkins
-The current folder (`/ci`) contains a separate playbook for setting up a Jenkins CI server.
+The current folder (`/ci`) contains a separate playbook for setting up a Jenkins CI server. The server will have the following things:
+- Jenkins (behind an nginx reverse proxy).
+- MySQL database for running tests against.
 
 **Note:** there is currently no support for running both this CI playbook and the base playbook against the same host(s).
 
@@ -15,13 +17,37 @@ The current folder (`/ci`) contains a separate playbook for setting up a Jenkins
 ## Deploy
 The following section describes the steps required to deploy Jenkins on a remote server.
 
+- Make sure you are in the `ci` directory.
+
+  ```bash
+  $ cd ci
+  ```
 - Set your hosts in the `hosts` file and make sure you can [connect to the servers via ssh](../docs/setting_up_ssh.md).
 - Copy the `variables.yml.dist` as `variables.yml`. Change the variables to the appropriate values where needed. [More info on these variables](roles/geerlingguy.jenkins/README.md).
 - Run playbook.
-  ```
+
+  ```bash
   $ ansible-playbook site.yml
   ```
 - Open a browser and navigate to http://SERVER_ADDRESS to go to the Jenkins dashboard.
+- Configure jenkins and jobs individually where needed (e.g. setup webhooks). 
+
+**Note for running tests:**   
+To be able to run `npm run test` for end-to-end tests in the jiskefet-api, a `.env` should contain the credentials to connect to the test database that is created by this playbook.
+
+`.env` and `ormconfig.json` need to be manually created on the server via ssh. The project is located in either `/var/lib/jenkins/jobs/<project_name>/workspace` (or `/var/lib/jenkins/workspace/<project_name>`). 
+
+You can copy the test env template for creating the `.env`:  
+```bash
+$ cp PATH_TO_PROJECT/<project_name>/environments/test.env.template .env
+```
+
+You can copy the `ormconfig.json.dist` for creating the `ormconfig.json`:
+```bash
+$ cp PATH_TO_PROJECT/<project_name>/ormconfig.json.dist .ormconfig.json
+```
+
+And then change the test db variables and the JWT secret in the `.env`, which are required for running tests against the db with authenticated API calls.
 
 [Back to table of contents](#Table-of-contents)
 
@@ -45,6 +71,7 @@ The general workflow for creating jobs:
     # When Jenkins runs locally
     $ cp -r /Users/Bob/.jenkins/jobs ~/projects/jiskefet-deploy/ci/files
     ```
+**WARNING**: Make sure to remove workspaces to ensure that `.env` variables are not being pushed by accident to GitHub. Workspaces exist in `jobs/<jobname>/workspace`.
 
 [Back to table of contents](#Table-of-contents)
 

ci/ansible.config.json.template

@@ -0,0 +1,14 @@
+{
+    "database_config": {
+        "type": "mysql",
+        "host": "",
+        "port": 3306,
+        "username": "",
+        "password": "",
+        "database": "",
+        "entities": [
+            "src/**/**.entity{.ts,.js}"
+        ],
+        "synchronize": true
+    }
+}

ci/files/create_db_charset_utf8mb4.sql

@@ -0,0 +1 @@
+CREATE DATABASE database_name CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci;

ci/files/my.cnf

@@ -0,0 +1,5 @@
+[client-server]
+
+[mysqld]
+character-set-client-handshake = FALSE
+character-set-server = utf8mb4

ci/hosts

@@ -1,2 +1,2 @@
 [ciservers]
-azure-jiskefet-ci
+vm-jiskefet-ci

ci/site.yml

@@ -17,6 +17,7 @@
   pre_tasks:
     # Open ports for Jenkins
     - include: tasks/firewall.yml
+    - include: ../tasks/create-jiskefet-user.yml
   roles:
     - geerlingguy.jenkins
   tasks:
@@ -28,4 +29,6 @@
     - include: tasks/git.yml
     # Install Node.js
     - include: ../tasks/nodejs.yml
+    # Install MySql (MariaDB)
+    - include: tasks/mysql.yml
 ...

ci/tasks/mysql.yml

@@ -0,0 +1,111 @@
+# /*
+#  * Copyright (C) 2018 Amsterdam University of Applied Sciences (AUAS)
+#  *
+#  * This software is distributed under the terms of the
+#  * GNU General Public Licence version 3 (GPL) version 3,
+#  * copied verbatim in the file "LICENSE"
+#  */
+
+--- # Install mariadb via ansible on centOS
+- name: Add official MariaDB repository
+  become: yes
+  become_user: root
+  yum_repository:
+    name: MariaDB
+    description: Official MariaDB repository
+    baseurl: "http://yum.mariadb.org/10.1/centos7-amd64"
+    gpgkey: https://yum.mariadb.org/RPM-GPG-KEY-MariaDB
+    gpgcheck: true
+  tags: mariadb
+
+- name: Install packages
+  become: yes
+  become_user: root
+  package:
+    name: "{{ item }}"
+    state: installed
+  with_items: 
+    - MariaDB
+    - MariaDB-server
+    - MySQL-python
+  tags: mariadb
+
+- name: Start mysql server and enable it on reboot
+  become: true
+  become_user: root
+  service: name=mariadb state=started enabled=true #debian: mysql
+
+- name: make sql directory at /var/lib/jiskefet
+  file:
+    path: /var/lib/jiskefet/sql
+    state: directory
+    owner: jiskefet-api
+
+- name: Copy my.cnf from local to remote
+  copy:
+    src: my.cnf
+    dest: /etc/my.cnf
+  become: true
+  become_method: su
+  become_user: root
+
+- name: Copy create_db_charset_utf8mb4.sql from local to remote
+  copy:
+    src: create_db_charset_utf8mb4.sql
+    dest: /var/lib/jiskefet/sql/create_db_charset_utf8mb4.sql
+  become: true
+  become_method: su
+  become_user: root
+
+- name: Replace create_db_charset_utf8mb4.sql database name with current name.
+  replace:
+    dest: /var/lib/jiskefet/sql/create_db_charset_utf8mb4.sql
+    regexp: 'database_name'
+    replace: '{{ database_config.database }}' 
+
+- name: Output ansible hostname
+  debug: msg="Hostname is {{ ansible_hostname }}"
+
+- name: Update mysql root password for all root accounts
+  become: yes
+  become_user: root
+  mysql_user:
+    name: "{{ database_config.username }}"
+    host: "{{ item }}"
+    password: "{{ database_config.password }}"
+    login_user: "{{ database_config.username }}"
+    login_password: "{{ database_config.password }}"
+    check_implicit_admin: yes
+    priv: "*.*:ALL,GRANT"
+  with_items:
+    - "{{ ansible_hostname }}"
+    - 127.0.0.1
+    - ::1
+    - localhost
+    - "%"
+
+- name: Execute sql file
+  mysql_db: 
+    login_user: '{{ database_config.username }}' 
+    login_password: '{{ database_config.password }}'
+    state: import
+    name: 'all'
+    target: /var/lib/jiskefet/sql/create_db_charset_utf8mb4.sql
+
+- name: Restart mysql
+  become: yes
+  become_user: root
+  service:
+    name: mysql
+    state: restarted
+
+- name: Add firewall exceptions
+  become: yes
+  become_user: root
+  command: firewall-cmd --permanent --add-service=mysql 
+
+- name: Reload firewall
+  become: yes
+  become_user: root
+  command: firewall-cmd --reload
+...

ci/variables.yml.dist

@@ -1,4 +1,12 @@
 ---
+# The config for the db that gets setup to run tests against
+database_config:
+  "host": "localhost"
+  "port": 3306
+  "username": "root"
+  "password": # Change this
+  "database": "jiskefet_test"
+
 # ------------------------------------------------------------------------------
 # These variables overwrite the vars used in the galaxy role geerlingguy.jenkins.
 # Location of the vars: