[CE-255] Allow ansible agent to avoid build any thing

Currently ansible agent requires download fabric source code to
at least build cryptogen and configtxgen binary to produce
org certificats. With this change, agent now can simply download
binaries and docker images from a configuration location such as
a location docker hub. This patch set also improves the docker
images download so that the not needed images will not be pulled
off of the hub. It also contains the recent changes on fabric
for peers to have a listen address and port to be configured.

Change-Id: Ic132fe8bc3ece96e6b99990664eec21b0b9cd568
Signed-off-by: tongliofcary <litong01@us.ibm.com>

src/agent/ansible/roles/deploy_compose/fabricbuild/tasks/apply.yml

@@ -27,67 +27,61 @@
     - "absent"
     - "directory"
 
-- name: Extract hyperledger fabric code
-  git:
-    repo: "{{ GIT_URL | default('http://gerrit.hyperledger.org/r/fabric') }}"
-    dest: "{{ fabricpath }}"
-    force: yes
-    refspec: "{{ GERRIT_REFSPEC | default('') }}"
-    version: "{{ (GERRIT_REFSPEC == '') | ternary('HEAD','FETCH_HEAD') }}"
-    depth: 1
-  tags: "code"
+- name: Download fabric docker images
+  include_tasks: "download.yml"
+  when: fabric.baseimage_tag|length > 0
+  tag: "downloadartifacts"
 
-- name: Make targets
-  shell: "make {{ item }} >> {{ gopath }}/build.log"
-  args:
-    chdir: "{{ fabricpath }}"
-  with_items: "{{ target.split(',') }}"
-  environment:
-    GOROOT: "{{ goroot }}"
-    GOPATH: "{{ gopath }}"
-    PATH: "{{ ansible_env.PATH}}:{{ goroot }}/bin"
-  tags: "make"
+- name: build fabric artifacts
+  include_tasks: "build.yml"
+  when: fabric.baseimage_tag|length == 0
+  tags: "buildartifacts"
 
-- name: Figure out the build tag
-  shell: "git rev-parse --short HEAD"
-  args:
-    chdir: "{{ fabricpath }}"
-  register: rawtag
+- name: Setup hyperledger directory
+  file:
+    path: "{{ fabricpath }}/build"
+    state: directory
+    force: yes
+    mode: 0775
 
-- name: Get base version
-  shell: grep '^BASE_VERSION' Makefile | cut -d '=' -f 2 | xargs
-  args:
-    chdir: "{{ fabricpath }}"
-  register: baseversion
+- stat:
+    path: "{{ fabricpath }}/build/bin/cryptogen"
+  register: binexists
 
-- name: Check if it is a release
-  shell: grep '^IS_RELEASE' Makefile | cut -d '=' -f 2 | xargs
-  args:
-    chdir: "{{ fabricpath }}"
-  register: isrelease
+- name: Download fabric binaries
+  unarchive:
+    src: "{{ fabric.repo.bin }}"
+    dest: "{{ fabricpath }}/build"
+    remote_src: yes
+  when: fabric.repo.bin|length > 0 and binexists.stat.exists == false
+  tags: "downloadbin"
 
-- name: Get the project version
-  set_fact:
-    project_version: >-
-      {{ (isrelease.stdout|lower == 'true') | ternary(baseversion.stdout,
-      baseversion.stdout+'-snapshot-'+rawtag.stdout) | trim }}
+- name: Validate the version, baseimage_tag and helper_tag to be deployed
+  fail:
+    msg: >-
+      the project version is empty, most likely configuration is not correct.
+      Please check baseimage_tag and helper_tag are set correct or you set
+      up to build fabric binaries and docker images!
+  when: >
+    project_version|length == 0 or baseimage_tag|length == 0 or
+    helper_tag|length == 0
 
-- name: Get all images for this build
+- name: Get all images for this deployment
   shell: >-
-    docker images --format {% raw %}'{{.Repository}}:{{.Tag }}' {% endraw %} |
-    grep {{ project_version }}
+    docker images --format {% raw %}'{{.Repository}}:{{.Tag }}' {% endraw %}
+    | grep 'hyperledger/fabric-' || :
   args:
     chdir: "{{ fabricpath }}"
   register: rawimages
-  when: fabric.baseimage_tag == ''
 
 - name: Save all the docker images
   shell: >-
     docker save -o {{ fabricworkdir }}/images/fabricimages.tar
     {{ rawimages.stdout_lines | join(' ') }}
   args:
     chdir: "{{ gopath }}"
-  when: fabric.baseimage_tag == '' and rawimages is defined and rawimages.stdout_lines|length > 0
+  when: rawimages.stdout_lines|length > 0 and fabric.baseimage_tag == ''
+  tags: "saveimages"
 
 - name: Create the project version file
   template:

src/agent/ansible/roles/deploy_compose/fabricbuild/tasks/build.yml

@@ -0,0 +1,67 @@
+---
+- name: Figuring out the server architecture
+  shell: uname -m
+  register: rawarch
+
+- name: Setup hyperledger directory
+  file:
+    path: "{{ fabricpath }}/build"
+    state: absent
+    force: yes
+    mode: 0775
+
+- name: Extract hyperledger fabric code
+  git:
+    repo: "{{ GIT_URL | default('http://gerrit.hyperledger.org/r/fabric') }}"
+    dest: "{{ fabricpath }}"
+    force: yes
+    refspec: "{{ GERRIT_REFSPEC | default('') }}"
+    version: "{{ (GERRIT_REFSPEC == '') | ternary('HEAD','FETCH_HEAD') }}"
+    depth: 1
+  tags: "extractcode"
+
+- name: Figure out the build tag
+  shell: "git rev-parse --short HEAD"
+  args:
+    chdir: "{{ fabricpath }}"
+  register: rawtag
+
+- name: Add docker targets if absent
+  set_fact:
+    target: "{{ target }},docker"
+  when: fabric.baseimage_tag|length == 0 and 'docker' not in target
+
+- name: Make targets
+  shell: "make {{ item }} >> {{ gopath }}/build.log"
+  args:
+    chdir: "{{ fabricpath }}"
+  with_items: "{{ target.split(',') }}"
+  environment:
+    GOROOT: "{{ goroot }}"
+    GOPATH: "{{ gopath }}"
+    PATH: "{{ ansible_env.PATH}}:{{ goroot }}/bin"
+  when: target | length > 0
+  tags: "makeimages"
+
+- name: Get base version
+  shell: grep '^BASE_VERSION' Makefile | cut -d '=' -f 2 | xargs
+  args:
+    chdir: "{{ fabricpath }}"
+  register: baseversion
+
+- name: Check if it is a release
+  shell: grep '^IS_RELEASE' Makefile | cut -d '=' -f 2 | xargs
+  args:
+    chdir: "{{ fabricpath }}"
+  register: isrelease
+
+- name: Get the project version
+  set_fact:
+    project_version: >-
+      {{ (isrelease.stdout|lower == 'true') | ternary(baseversion.stdout,
+      baseversion.stdout+'-snapshot-'+rawtag.stdout) | trim }}
+
+- name: Set project version and container tags
+  set_fact:
+    baseimage_tag: "{{ rawarch.stdout }}-{{ project_version }}"
+    helper_tag: "{{ fabric.helper_tag }}"

src/agent/ansible/roles/deploy_compose/fabricbuild/tasks/destroy.yml

@@ -20,4 +20,5 @@
     GOROOT: "{{ goroot }}"
     GOPATH: "{{ gopath }}"
     PATH: "{{ ansible_env.PATH}}:{{ goroot }}/bin"
+  when: fabric.baseimage_tag|length == 0
   tags: "clean"

src/agent/ansible/roles/deploy_compose/fabricbuild/tasks/download.yml

@@ -0,0 +1,14 @@
+---
+- name: Try to get the project version from the baseimage_tag
+  set_fact:
+    project_version: >-
+      {{ fabric.baseimage_tag |
+         regex_replace('.*([0-9]{1,2}\.[0-9]{1,2}\.[0-9]{1,2}).*', '\1') }}
+    baseimage_tag: "{{ fabric.baseimage_tag }}"
+    helper_tag: "{{ fabric.helper_tag | default(fabric.baseimage_tag) }}"
+
+- name: Set the version to 1.1.0 if the tag is a commit hashcode
+  set_fact:
+    project_version: >-
+      {{ fabric.baseimage_tag | match(project_version) |
+         ternary('1.1.0', project_version) }}

src/agent/ansible/roles/deploy_compose/fabricbuild/templates/VERSION.j2

@@ -1 +1 @@
-{{ project_version }}
+{{  baseimage_tag }}

src/agent/ansible/roles/deploy_compose/fabricsetup/tasks/apply.yml

@@ -1,14 +1,9 @@
 ---
-- name: Figuring out the server architecture
-  shell: uname -m
-  register: rawarch
-
 - name: Setup and initialize variables
   set_fact:
     current_host: "{{ hostvars[inventory_hostname].inter_name }}"
     fabricworkdir: "/opt/gopath/{{ env }}/fabric"
     gopath: "/opt/gopath/{{ env }}"
-    arch: "{{ rawarch.stdout}}"
     peers: "{{ [] }}"
     orderers: "{{ [] }}"
     cas: "{{ [] }}"
@@ -29,8 +24,6 @@
     allpeers: "{{ [] }}"
     clihost: ""
     filterstr: ""
-    thetag: "{{ rawarch.stdout + '-' + fabric.baseimage_tag }}"
-    helpertag: "{{ rawarch.stdout + '-' + fabric.helper_tag }}"
 
 - name: Make sure that working directory is clean
   become: true
@@ -143,23 +136,25 @@
     scp -i "/opt/gopath/id_rsa" -r -o "StrictHostKeyChecking no" "{{ fabric.
     ssh_user }}@{{ hostvars[groups['builders'][0]].private_ip }}:{{ fabricworkdir }}/images/VERSION"
     "{{ fabricworkdir }}/VERSION"
-  when: fabric.baseimage_tag == ''
-
-- name: Load all the docker images created by build machine
-  shell: >-
-    docker load -i {{ fabricworkdir }}/fabricimages.tar
-  when: fabric.baseimage_tag == '' and inventory_hostname not in groups['builders']
 
 - name: Find out the image tags
   slurp:
     src: "{{ fabricworkdir }}/VERSION"
-  register: project_version
-  when: fabric.baseimage_tag == ''
+  register: imagetag
 
-- name: Set the container tag
+- name: Set image tag from the file
   set_fact:
-    thetag: "{{ arch + '-' + (project_version['content'] | b64decode) }}"
-  when: fabric.baseimage_tag == ''
+    thetag: "{{ imagetag['content'] | b64decode }}"
+    helpertag: "{{ fabric.helper_tag }}"
+
+- stat:
+    path: "{{ fabricworkdir }}/fabricimages.tar"
+  register: imagepack
+
+- name: Load all the docker images created by build machine
+  shell: >-
+    docker load -i {{ fabricworkdir }}/fabricimages.tar
+  when: imagepack.stat.exists == true and inventory_hostname not in groups['builders']
 
 - name: Unpack the certificates
   unarchive:
@@ -199,17 +194,19 @@
   set_fact:
     orgmembers: "{{ peers | map(attribute='org') | list  | unique | sort | join(\".member' '\") | trim | replace(' ', ',') }}"
 
-- name: Pull container images from the docker hub
-  command: "docker pull {{ item }}"
-  when: fabric.baseimage_tag | length > 0
+- name: Pull necessary container images from the docker hub
+  command: "docker pull {{ fabric.repo.url }}{{ item.name }}"
+  when: item.flag | length > 0 and fabric.baseimage_tag | length > 0
   with_items:
-    - "hyperledger/fabric-tools:{{ thetag }}"
-    - "hyperledger/fabric-ccenv:{{ thetag }}"
-    - "hyperledger/fabric-kafka:{{ thetag }}"
-    - "hyperledger/fabric-zookeeper:{{ thetag }}"
-    - "hyperledger/fabric-peer:{{ thetag }}"
-    - "hyperledger/fabric-orderer:{{ thetag }}"
-    - "hyperledger/fabric-couchdb:{{ thetag }}"
+    - { name: "fabric-ca:{{ helpertag }}", flag: "{{ cas }}" }
+    - { name: "fabric-zookeeper:{{ helpertag }}", flag: "{{ zookeepers }}" }
+    - { name: "fabric-kafka:{{ helpertag }}", flag: "{{ kafkas }}" }
+    - { name: "fabric-couchdb:{{ helpertag }}", flag: "{{ peers }}" }
+    - { name: "fabric-orderer:{{ thetag }}", flag: "{{ orderers }}" }
+    - { name: "fabric-peer:{{ thetag }}", flag: "{{ peers }}" }
+    - { name: "fabric-ccenv:{{ thetag }}", flag: "{{ peers }}" }
+    - { name: "fabric-tools:{{ thetag }}", flag: "tools" }
+  tags: "pullimages"
 
 - name: Create docker compose files
   template:

src/agent/ansible/roles/deploy_compose/fabricsetup/templates/ca-compose.j2

@@ -6,7 +6,7 @@ services:
 {% for ca in cas %}
   {{ ca.name }}:
     container_name: {{ ca.name}}
-    image: hyperledger/fabric-ca:{{ (fabric.helper_tag == "") | ternary(thetag, helpertag) }}
+    image: {{ fabric.repo.url }}fabric-ca:{{ helpertag | default(thetag) }}
     network_mode: bridge
     hostname: {{ ca.name}}
     environment:

src/agent/ansible/roles/deploy_compose/fabricsetup/templates/cli-compose.j2

@@ -5,7 +5,7 @@ services:
 
   fabriccli:
     container_name: fabriccli
-    image: hyperledger/fabric-tools:{{ thetag }}
+    image: {{ fabric.repo.url }}fabric-tools:{{ thetag }}
     network_mode: bridge
     hostname: fabriccli
     environment:

src/agent/ansible/roles/deploy_compose/fabricsetup/templates/kafka-compose.j2

@@ -6,7 +6,7 @@ services:
 {% for kafka in kafkas %}
   {{ kafka }}:
     container_name: {{ kafka }}
-    image: hyperledger/fabric-kafka:{{ (fabric.helper_tag == "") | ternary(thetag, helpertag) }}
+    image: {{ fabric.repo.url }}fabric-kafka:{{ helpertag | default(thetag) }}
     restart: always
     network_mode: bridge
     hostname: {{ kafka }}

src/agent/ansible/roles/deploy_compose/fabricsetup/templates/orderer-compose.j2

@@ -6,7 +6,7 @@ services:
 {% for orderer in orderers %}
   {{ orderer.name }}:
     container_name: {{ orderer.name }}
-    image: hyperledger/fabric-orderer:{{ thetag }}
+    image: {{ fabric.repo.url }}fabric-orderer:{{ thetag }}
     network_mode: bridge
     hostname: {{ orderer.name }}
     environment:

src/agent/ansible/roles/deploy_compose/fabricsetup/templates/peer-compose.j2

@@ -7,14 +7,14 @@ services:
 {% if fabric.peer_db == 'CouchDB' %}
   couchdb-{{ peer.name }}:
     container_name: couchdb-{{ peer.name }}
-    image: hyperledger/fabric-couchdb:{{ (fabric.helper_tag == "") | ternary(thetag, helpertag) }}
+    image: {{ fabric.repo.url }}fabric-couchdb:{{ helpertag | default(thetag) }}
     network_mode: bridge
     hostname: couchdb-{{ peer.name }}
 {% endif %}
 
   {{ peer.name }}:
     container_name: {{ peer.name }}
-    image: hyperledger/fabric-peer:{{ thetag }}
+    image: {{ fabric.repo.url }}fabric-peer:{{ thetag }}
     network_mode: bridge
     hostname: {{ peer.name }}
     environment:

src/agent/ansible/roles/deploy_compose/fabricsetup/templates/zookeeper-compose.j2

@@ -6,7 +6,7 @@ services:
 {% for zookeeper in zookeepers %}
   {{ zookeeper }}:
     container_name: {{ zookeeper }}
-    image: hyperledger/fabric-zookeeper:{{ (fabric.helper_tag == "") | ternary(thetag, helpertag) }}
+    image: {{ fabric.repo.url }}fabric-zookeeper:{{ helpertag | default(thetag) }}
     network_mode: bridge
     hostname: {{ zookeeper }}
     restart: always

src/agent/ansible/roles/deploy_compose/plays.yml

@@ -6,7 +6,7 @@
   vars_files:
     - "./../../vars/{{ env }}.yml"
   tasks:
-    - include: "fabricbuild/tasks/{{ mode }}.yml"
+    - include_tasks: "fabricbuild/tasks/{{ mode }}.yml"
   tags: "fabricbuild"
 
 - name: generate certificates