Update test debugging

Signed-off-by: johnsonw <wjohnson@whamcloud.com>

docker/docker-compose.yml

@@ -170,6 +170,8 @@ services:
  context: .
  dockerfile: ./corosync.dockerfile
  deploy: *default-deploy
+ environment:
+ - LOG_LEVEL=20
  volumes:
  - "manager-config:/var/lib/chroma"
  gunicorn:
@@ -182,6 +184,7 @@ services:
  environment:
  - USE_CONSOLE=1
  - PROXY_HOST=gunicorn
+ - LOG_LEVEL=20
  volumes:
  - "manager-config:/var/lib/chroma"
  http-agent:
@@ -195,6 +198,7 @@ services:
  deploy: *default-deploy
  environment:
  - PROXY_HOST=http-agent
+ - LOG_LEVEL=20
  iml-agent-comms:
  image: "imlteam/iml-agent-comms:6.2.0-dev"
  volumes:
@@ -223,13 +227,16 @@ services:
  - DEVICE_AGGREGATOR_URL=http://device:8008
  - PROXY_HOST=iml-action-runner
  - INFLUXDB_SERVER_FQDN=influxdb
+ - LOG_LEVEL=20
  lustre-audit:
  image: "imlteam/manager-lustre-audit:6.2.0-dev"
  hostname: "lustre-audit"
  build:
  context: .
  dockerfile: ./lustre-audit.dockerfile
  deploy: *default-deploy
+ environment:
+ - LOG_LEVEL=20
  volumes:
  - "manager-config:/var/lib/chroma"
  plugin-runner:
@@ -243,13 +250,16 @@ services:
  - "manager-config:/var/lib/chroma"
  environment:
  - DEVICE_AGGREGATOR_URL=http://device:8008
+ - LOG_LEVEL=20
  power-control:
  image: "imlteam/manager-power-control:6.2.0-dev"
  hostname: "power-control"
  build:
  context: .
  dockerfile: ./power-control.dockerfile
  deploy: *default-deploy
+ environment:
+ - LOG_LEVEL=20
  volumes:
  - "manager-config:/var/lib/chroma"
  syslog:
@@ -259,6 +269,8 @@ services:
  context: .
  dockerfile: ./syslog.dockerfile
  deploy: *default-deploy
+ environment:
+ - LOG_LEVEL=20
  volumes:
  - "manager-config:/var/lib/chroma"
  iml-warp-drive:
@@ -314,6 +326,7 @@ services:
  - "manager-config:/var/lib/chroma"
  environment:
  - SERVER_HTTP_URL=gunicorn
+ - RUST_LOG=info
  iml-ostpool:
  image: "imlteam/iml-ostpool:6.2.0-dev"
  hostname: "iml-ostpool"
@@ -332,6 +345,8 @@ services:
  context: ../
  dockerfile: ./docker/iml-mailbox.dockerfile
  deploy: *default-deploy
+ environment:
+ - RUST_LOG=info
  volumes:
  - "mailbox:/var/spool/iml/mailbox"
  - "/sys/fs/cgroup:/sys/fs/cgroup:ro"
@@ -360,13 +375,16 @@ services:
  - "manager-config:/var/lib/chroma"
  environment:
  - PROXY_HOST=iml-postoffice
+ - RUST_LOG=info
  iml-timer:
  image: "imlteam/iml-timer:6.2.0-dev"
  hostname: "iml-timer"
  build:
  context: ../
  dockerfile: ./docker/iml-timer.dockerfile
  deploy: *default-deploy
+ environment:
+ - RUST_LOG=info
  volumes:
  - "/sys/fs/cgroup:/sys/fs/cgroup:ro"
  - "timer_run:/run"
@@ -389,6 +407,7 @@ services:
  - USE_STRATAGEM
  - EXA_VERSION
  - INFLUXDB_IML_STATS_LONG_DURATION
+ - LOG_LEVEL=20
  deploy:
  restart_policy:
  condition: on-failure

iml-system-test-utils/src/lib.rs

@@ -273,14 +273,22 @@ impl Config {
  pub fn get_setup_config(&self) -> String {
  match &self.test_type {
  TestType::Rpm => format!(
- r#"USE_STRATAGEM = {}
-BRANDING = "{}""#,
+ r#"USE_STRATAGEM={}
+BRANDING={}
+LOG_LEVEL=10
+RUST_LOG=debug
+NTP_SERVER_HOSTNAME=adm.local
+"#,
  if self.use_stratagem { "True" } else { "False" },
  self.branding.to_string()
  ),
  TestType::Docker => format!(
  r#"USE_STRATAGEM={}
-BRANDING={}"#,
+BRANDING={}
+LOG_LEVEL=10
+RUST_LOG=debug
+NTP_SERVER_HOSTNAME=10.73.10.1
+"#,
  self.use_stratagem,
  self.branding.to_string()
  ),
@@ -447,7 +455,7 @@ pub async fn configure_iml(config: Config) -> Result<Config, CmdError> {
 pub async fn deploy_servers(config: Config) -> Result<Config, CmdError> {
  for (profile, hosts) in &config.profile_map {
  let host_ips = config.hosts_to_ips(&hosts);
- for host in host_ips {
+ for host in &host_ips {
  tracing::debug!("pinging host to make sure it is up.");
  ssh::ssh_exec_cmd(host, "uname -r")
  .await?
@@ -463,6 +471,7 @@ pub async fn deploy_servers(config: Config) -> Result<Config, CmdError> {
  };
 
  ssh::add_servers(&config.manager_ip, profile, hosts).await?;
+ ssh::enable_debug_on_hosts(&host_ips).await?;
  }
 
  vagrant::halt()
@@ -616,12 +625,12 @@ pub async fn configure_rpm_setup(config: &Config) -> Result<(), CmdError> {
 
  let vagrant_path = canonicalize("../vagrant/").await?;
  let mut config_path = vagrant_path.clone();
- config_path.push("local_settings.py");
+ config_path.push("overrides.conf");
 
  let mut file = File::create(config_path).await?;
  file.write_all(config_content.as_bytes()).await?;
 
- let mut vm_cmd: String = "sudo cp /vagrant/local_settings.py /usr/share/chroma-manager/".into();
+ let mut vm_cmd: String = "mkdir -p /var/lib/chroma && sudo cp /vagrant/overrides.conf /var/lib/chroma".into();
  if config.use_stratagem {
  let mut server_profile_path = vagrant_path.clone();
  server_profile_path.push("stratagem-server.profile");

iml-system-test-utils/src/ssh.rs

@@ -216,6 +216,12 @@ pub async fn wait_for_ntp_for_adm<'a, 'b>(
  ssh_script_parallel(hosts, "scripts/wait_for_ntp.sh", &["adm.local"]).await
 }
 
+pub async fn enable_debug_on_hosts<'a, 'b>(
+ hosts: &'b [&'a str],
+) -> Result<Vec<(&'a str, Output)>, CmdError> {
+ ssh_script_parallel(hosts, "scripts/enable_debug.sh", &[]).await
+}
+
 pub async fn create_iml_diagnostics<'a, 'b>(
  hosts: Vec<&'a str>,
  prefix: &'a str,

vagrant/Vagrantfile

@@ -467,6 +467,10 @@ Vagrant.configure('2') do |config|
  mkdir -p /mnt/mdt{0,1}
  SHELL
 
+ mds.vm.provision 'enable-debug',
+ type: 'shell',
+ run: 'never',
+ path: 'scripts/enable_debug.sh'
  end
  end
 
@@ -793,6 +797,10 @@ Vagrant.configure('2') do |config|
  mkdir -p /mnt/ost{0..9}
  SHELL
 
+ oss.vm.provision 'enable-debug',
+ type: 'shell',
+ run: 'never',
+ path: 'scripts/enable_debug.sh'
  end
  end
 
@@ -833,6 +841,10 @@ Vagrant.configure('2') do |config|
  type: 'shell',
  run: 'never',
  path: './scripts/configure_lustre_network.sh'
+ c.vm.provision 'enable-debug',
+ type: 'shell',
+ run: 'never',
+ path: 'scripts/enable_debug.sh'
 
  end
  end

vagrant/scripts/enable_debug.sh

@@ -0,0 +1,2 @@
+#!/bin/bash
+touch /tmp/chroma-agent-debug

vagrant/scripts/wait_for_ntp.sh

@@ -7,7 +7,7 @@ systemctl stop ntpd.service
 sntp -s "$NTP_SERVER"
 sntp -s "$NTP_SERVER"
 
-systemctl restart ntpd.service
+systemctl start ntpd.service
 
 until ntpstat;
 do