Fix more ansible 2.8.7 compability issues (#1267)

* Fix more ansible 2.8.7 compability issues

Issues fixed in this commit:
* Change 'connection: local' to 'delegate_to: localhost' to support configuring ansible_python_interpreter for specific test servers
* Add pip_executable configuration option
* Fix datetime call in kickstart.py
* Fix tags issue of fanout deploy/config
* Remove uncesessary tags in vlantb and everflow_testbed testing
* Fix some 'with_items' and 'with_dict'
* New ansible parses inventory file as full path, fix issue of finding correct inventory file in pytest_runner.yml
* Fix issue of creating PTF host in conftest

Change-Id: I245ff5c4b2480a9b8153c0b896c7a25a448d5956
Signed-off-by: Xin Wang <xinw@mellanox.com>

ansible/config_sonic_basedon_testbed.yml

@@ -42,7 +42,7 @@
 
     - name: Gathering testbed information
       test_facts: testbed_name="{{ testbed_name }}" testbed_file="{{ testbed_file }}"
-      connection: local
+      delegate_to: localhost
 
     - fail: msg="The DUT you are trying to run test does not belongs to this testbed"
       when: testbed_facts['dut'] != inventory_hostname
@@ -58,15 +58,15 @@
     when: testbed_name is defined
 
   - topo_facts: topo={{ topo }}
-    connection: local
+    delegate_to: localhost
 
   - set_fact:
       VM_topo: "{% if 'ptf' in topo %}False{% else %}True{% endif %}"
       remote_dut: "{{ ansible_ssh_host }}"
 
   - name: gather testbed VM informations
     testbed_vm_info: base_vm={{ testbed_facts['vm_base'] }} topo={{ testbed_facts['topo'] }}
-    connection: local
+    delegate_to: localhost
     when: "VM_topo | bool"
 
   - name: find interface name mapping and individual interface speed if defined
@@ -100,7 +100,7 @@
   - name: create minigraph file in ansible minigraph folder
     template: src=templates/minigraph_template.j2
               dest=minigraph/{{ inventory_hostname}}.{{ topo }}.xml
-    connection: local
+    delegate_to: localhost
     when: local_minigraph is defined and local_minigraph|bool == true
 
   - block:

ansible/fanout_connect.yml

@@ -4,7 +4,7 @@
   gather_facts: no
   tasks:
       - fail: msg="Please provide VM server name and server port name, see comment line in playbook"
-        when: 
+        when:
             - dut is not defined
 
       - block:
@@ -16,7 +16,7 @@
 
         - name: get the username running the deploy
           command: whoami
-          connection: local
+          delegate_to: localhost
           become: no
           register: calling_username
           changed_when: false

ansible/inventory

@@ -1,6 +1,6 @@
 [sonic_latest]
 switch1  ansible_host=10.0.0.100  sonic_version=v2  sonic_hwsku=Force10-S6000
-switch2  ansible_host=10.0.0.101  sonic_version=v2  sonic_hwsku=ACS-MSN2700 
+switch2  ansible_host=10.0.0.101  sonic_version=v2  sonic_hwsku=ACS-MSN2700
 switch3  ansible_host=10.0.0.102  sonic_version=v2  sonic_hwsku=Force10-S6000   # LAG topo: 8 LAGs x 2 members/lag to spines; 16 ports to Tors
 switch4  ansible_host=10.0.0.103  sonic_version=v2  sonic_hwsku=AS7512 sonic_portsku=32x40
 switch5  ansible_host=10.0.0.104  sonic_version=v2  sonic_hwsku=ACS-MSN2700 # LAG topo: 8 LAGs x 2 members/lag to spines; 16 ports to Tors

ansible/library/lldp_facts.py

@@ -57,7 +57,7 @@
 EXAMPLES = '''
 # Gather LLDP facts with SNMP version 2
 - snmp_facts: host={{ inventory_hostname }} version=2c community=public
-  connection: local
+  delegate_to: localhost
 
 # Gather LLDP facts using SNMP version 3
 - lldp_facts:
@@ -142,7 +142,7 @@ def main():
         supports_check_mode=False)
 
     m_args = module.params
-        
+
     if not has_pysnmp:
         module.fail_json(msg='Missing required pysnmp module (check docs)')
 
@@ -152,14 +152,14 @@ def main():
     if m_args['version'] == "v2" or m_args['version'] == "v2c":
         if not m_args['community']:
             module.fail_json(msg='Community not set when using snmp version 2')
-            
+
     if m_args['version'] == "v3":
         if m_args['username'] is None:
             module.fail_json(msg='Username not set when using snmp version 3')
 
         if m_args['level'] == "authPriv" and m_args['privacy'] == None:
             module.fail_json(msg='Privacy algorithm not set when using authPriv')
-            
+
         if m_args['integrity'] == "sha":
             integrity_proto = cmdgen.usmHMACSHAAuthProtocol
         elif m_args['integrity'] == "md5":
@@ -169,7 +169,7 @@ def main():
             privacy_proto = cmdgen.usmAesCfb128Protocol
         elif m_args['privacy'] == "des":
             privacy_proto = cmdgen.usmDESPrivProtocol
-    
+
     # Use SNMP Version 2
     if m_args['version'] == "v2" or m_args['version'] == "v2c":
         snmp_auth = cmdgen.CommunityData(m_args['community'])
@@ -188,7 +188,7 @@ def main():
     v = DefineOid(dotprefix=False)
 
     Tree = lambda: defaultdict(Tree)
-                               
+
     results = Tree()
 
     host = m_args['host']
@@ -222,9 +222,9 @@ def main():
     lldp_rem_port_desc = dict()
     lldp_rem_chassis_id = dict()
     lldp_rem_sys_desc = dict()
-    
+
     vbd = []
-    
+
     for var_binds in var_table:
         for oid, val in var_binds:
             current_oid = oid.prettyPrint()
@@ -233,7 +233,7 @@ def main():
             vbd.append(current_val)
 
             try:
-                if_name = inverse_if_table[str(current_oid.split(".")[-2])] 
+                if_name = inverse_if_table[str(current_oid.split(".")[-2])]
             except Exception as e:
                 print json.dumps({
                     "unbound_interface_index": str(current_oid.split(".")[-2])
@@ -259,10 +259,10 @@ def main():
     lldp_data = dict()
 
     for intf in lldp_rem_sys.viewkeys():
-        lldp_data[intf] = {'neighbor_sys_name': lldp_rem_sys[intf], 
-                                'neighbor_port_desc': lldp_rem_port_desc[intf], 
-                                'neighbor_port_id': lldp_rem_port_id[intf], 
-                                'neighbor_sys_desc': lldp_rem_sys_desc[intf], 
+        lldp_data[intf] = {'neighbor_sys_name': lldp_rem_sys[intf],
+                                'neighbor_port_desc': lldp_rem_port_desc[intf],
+                                'neighbor_port_id': lldp_rem_port_id[intf],
+                                'neighbor_sys_desc': lldp_rem_sys_desc[intf],
                                 'neighbor_chassis_id': lldp_rem_chassis_id[intf]}
 
 

ansible/library/testing_port_ip_facts.py

@@ -33,7 +33,7 @@
     dut_switch_ports: "{{ dut_switch_ports }}"
     minigraph_bgp: "{{ minigraph_bgp }}"
     minigraph_neighbors: "{{ minigraph_neighbors }}"
-  connection: local
+  delegate_to: localhost
 '''
 
 

ansible/linkstate/up.yml

@@ -42,10 +42,10 @@
         src: "{{ item }}"
         dest: /root
       with_items:
-        - ../files/lab_connection_graph.xml
-        - ../veos
-        - scripts/ptf_proxy.py
-        - ../vars/topo_{{ topo }}.yml
+        - "../files/lab_connection_graph.xml"
+        - "../veos"
+        - "scripts/ptf_proxy.py"
+        - "../vars/topo_{{ topo }}.yml"
       ignore_errors: yes     # either sonic_str_*.csv or sonic_lab_*.csv exists
     - name: Rename topo to common filename
       command: mv topo_{{ topo }}.yml topo.yaml

ansible/roles/fanout/tasks/fanout_mlnx.yml

@@ -28,18 +28,29 @@
     vars:
       action_variable: "deploy"
     when: peer_hwsku == "MLNX-OS"
-    tags: deploy
+  tags: deploy
 
  ###################################################################
  # build, deploy and start docker images for the PFC WD test       #
  ###################################################################
-- include_tasks: mlnx/deploy_pfcwd_fanout.yml
+- name: build, deploy and start docker images for the PFC WD test
+  include_tasks:
+    file: mlnx/deploy_pfcwd_fanout.yml
+    apply:
+      tags:
+        - deploy
+        - pfcwd_config
   when: peer_hwsku == "MLNX-OS"
   tags: deploy,pfcwd_config
 
  ###################################################################
  # check and recover docker images for the PFC WD test             #
  ###################################################################
-- include_tasks: mlnx/check_pfcwd_fanout.yml
+- name: check and recover docker images for the PFC WD test
+  include_tasks:
+    file: mlnx/check_pfcwd_fanout.yml
+    apply:
+      tags:
+        - check_pfcwd_config
   when: peer_hwsku == "MLNX-OS"
   tags: check_pfcwd_config

ansible/roles/fanout/tasks/main.yml

@@ -8,27 +8,31 @@
 ################################################################################################
 - name: Gathering lab graph facts about the device
   conn_graph_facts: host={{ inventory_hostname }}
-  connection: local
+  delegate_to: localhost
+  tags: always
 
 - set_fact: sw_type="{{ device_info['Type'] }}"
 
 - set_fact: os='eos'
   when: os is not defined
+  tags: always
 
-- include_tasks: fanout_eos.yml
+- import_tasks: fanout_eos.yml
   when: os == 'eos'
 
-- include_tasks: fanout_sonic.yml
+- import_tasks: fanout_sonic.yml
   when: os == 'sonic'
 
-- include_tasks: fanout_mlnx.yml
+- import_tasks: fanout_mlnx.yml
   when: os == 'mellanox'
 
 - block:
   - set_fact:
       leaf_name: "{{ inventory_hostname }}"
       leaf: "{{ ansible_host }}"
 
-  - include_tasks: rootfanout_connect.yml
-      deploy_leaf=true
+  - import_tasks: rootfanout_connect.yml
+    vars:
+      deploy_leaf: true
   when: sw_type == 'FanoutLeaf'
+  tags: always

ansible/roles/fanout/tasks/mlnx/deploy_pfcwd_fanout.yml

@@ -32,7 +32,6 @@
 - name: Download pre-built pfcwd dockers if path specified
   include_tasks: download_copy_pfcwd_fanout.yml
   with_items: "{{ pfcwd_dockers }}"
-  delegate_to: localhost
   when: pfcwd_dockers_url is defined
 
 - name: Load and start dockers

ansible/roles/fanout/tasks/mlnx/download_copy_pfcwd_fanout.yml

@@ -8,6 +8,7 @@
 
     - name: Download pre-built pfcwd docker image
       get_url: url={{ pfcwd_dockers_url }}/{{ item | basename }} dest={{ filename }}
+      delegate_to: localhost
 
     - name: Copy the downloaded pfcwd docker image to switch
       include_tasks: scp_copy.yml
@@ -19,3 +20,4 @@
 
     - name: Remove the downloaded pfcwd docker image
       file: path={{ filename }} state=absent
+      delegate_to: localhost

ansible/roles/fanout/tasks/rootfanout_connect.yml

@@ -9,13 +9,13 @@
 
 - name: Gathering connection facts about the DUT or leaffanout device
   conn_graph_facts: host={{ dut }}
-  connection: local
+  delegate_to: localhost
   tags: always
   register: devinfo
 
-- name: Gathering connection facts about the lab 
-  conn_graph_facts: 
-  connection: local
+- name: Gathering connection facts about the lab
+  conn_graph_facts:
+  delegate_to: localhost
   tags: always
   register: lab
 
@@ -26,11 +26,11 @@
 - name: Find the root fanout switch
   set_fact:
     ansible_host: "{{ lab_devices[item]['mgmtip'] }}"
-    root_dev: "{{ item }}" 
+    root_dev: "{{ item }}"
   with_items: "{{ lab_devices }}"
   when: lab_devices[item]['Type'] == 'FanoutRoot'
 
-- set_fact: 
+- set_fact:
     root_conn: "{{ lab.ansible_facts['device_conn'][root_dev] }}"
 
 - name: Change root fanout port vlan

ansible/roles/sonic-common/tasks/snmp.yml

@@ -42,13 +42,13 @@
   template: src=snmp.yml.j2
             dest=/etc/sonic/snmp.yml
             mode=0644
-    
+
 - name: Setup sonic_version file (for oneimage snmp dockers)
   become: true
   template: src=sonic_version.yml.j2
             dest=/etc/sonic/sonic_version.yml
             mode=0644
-    
+
 - name: Setup sysDescription file (for legacy snmp dockers)
   become: true
   template: src=sysDescription.j2
@@ -60,7 +60,7 @@
 - name: Determine if alias mappings are required
   stat: path="roles/sonicv2/files/ssw/{{ sonic_hwsku }}/alias_map.json"
   become: false
-  connection: local
+  delegate_to: localhost
   register: snmp_remap
 
 - block:

ansible/roles/test/tasks/advanced-reboot.yml

@@ -4,7 +4,7 @@
 - block:
     - name: figure out vm hosts
       testbed_vm_info: base_vm={{ vm }} topo={{ testbed_type }}
-      connection: local
+      delegate_to: localhost
 
     - set_fact:
         vm_hosts: "{{ neighbor_eosvm_mgmt.values() }}"

ansible/roles/test/tasks/advanced_reboot/reboot-image-handle.yml

@@ -38,7 +38,7 @@
 
     - name: 'Setup restoring initial image {{ current_sonic_image }}'
       shell: /bin/true
-      connection: local
+      delegate_to: localhost
       notify:
       - restore current image
       - reboot sonic

ansible/roles/test/tasks/bgp_entry_flap.yml

@@ -19,13 +19,13 @@
 
 - block:
   - name: Assert the particular entry is in nexthopgroup table
-    assert: 
+    assert:
       that: nexthop[addr] in nexthopgroup[item]
     with_items: "{{ nexthopgroup }}"
 
   - name: Gathering minigraph facts about neighbor
     minigraph_facts: host={{ name }} filename="{{ vmhost_num }}-{{ name }}.xml"
-    connection: local
+    delegate_to: localhost
     become: no
 
   - name: Shut down BGP session from neighbor
@@ -47,7 +47,7 @@
     vars:
       ansible_shell_type: docker
       ansible_python_interpreter: docker exec -i sswsyncd python
-  
+
   - name: Poll for updated tables until peer is not in nexthop groups
     switch_tables: asic="{{asic}}" nexthop=yes nexthopgroup=yes
     become: yes

ansible/roles/test/tasks/bgp_gr_helper/get_vm_info.yml

@@ -1,6 +1,6 @@
 - name: Gathering lab graph facts about the device
   conn_graph_facts: host={{ ansible_host }}
-  connection: local
+  delegate_to: localhost
   tags: always
 
 - name: Init variables.

ansible/roles/test/tasks/check_fanout_interfaces.yml

@@ -1,7 +1,7 @@
 - block:
   - name: Gathering lab graph facts about the device
     conn_graph_facts: host={{ inventory_hostname }}
-    connection: local
+    delegate_to: localhost
 
   - name: Fanout hostname
     set_fact: fanout_switch={{ device_conn['Ethernet0']['peerdevice'] }}

ansible/roles/test/tasks/check_sw_vm_interfaces.yml

@@ -23,7 +23,7 @@
 
   - name: Gathering testbed information
     test_facts: testbed_name="{{ testbed_name }}"
-    connection: local
+    delegate_to: localhost
     ignore_errors: yes
 
   - name: Gather vm list from Testbed server