From 2913f1023f53dcda3eaac0244508dfa9ca5632f0 Mon Sep 17 00:00:00 2001 From: johnsonw Date: Mon, 13 Jul 2020 15:19:54 -0400 Subject: [PATCH] Update test debugging Signed-off-by: johnsonw --- docker/docker-compose.yml | 19 +++++++++++++++++++ iml-system-test-utils/src/lib.rs | 21 +++++++++++++++------ iml-system-test-utils/src/ssh.rs | 6 ++++++ vagrant/Vagrantfile | 12 ++++++++++++ vagrant/scripts/enable_debug.sh | 2 ++ vagrant/scripts/wait_for_ntp.sh | 2 +- 6 files changed, 55 insertions(+), 7 deletions(-) create mode 100644 vagrant/scripts/enable_debug.sh diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index ad0d8ff5e6..54a4f1ed77 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -170,6 +170,8 @@ services: context: . dockerfile: ./corosync.dockerfile deploy: *default-deploy + environment: + - LOG_LEVEL=20 volumes: - "manager-config:/var/lib/chroma" gunicorn: @@ -182,6 +184,7 @@ services: environment: - USE_CONSOLE=1 - PROXY_HOST=gunicorn + - LOG_LEVEL=20 volumes: - "manager-config:/var/lib/chroma" http-agent: @@ -195,6 +198,7 @@ services: deploy: *default-deploy environment: - PROXY_HOST=http-agent + - LOG_LEVEL=20 iml-agent-comms: image: "imlteam/iml-agent-comms:6.2.0-dev" volumes: @@ -223,6 +227,7 @@ services: - DEVICE_AGGREGATOR_URL=http://device:8008 - PROXY_HOST=iml-action-runner - INFLUXDB_SERVER_FQDN=influxdb + - LOG_LEVEL=20 lustre-audit: image: "imlteam/manager-lustre-audit:6.2.0-dev" hostname: "lustre-audit" @@ -230,6 +235,8 @@ services: context: . dockerfile: ./lustre-audit.dockerfile deploy: *default-deploy + environment: + - LOG_LEVEL=20 volumes: - "manager-config:/var/lib/chroma" plugin-runner: @@ -243,6 +250,7 @@ services: - "manager-config:/var/lib/chroma" environment: - DEVICE_AGGREGATOR_URL=http://device:8008 + - LOG_LEVEL=20 power-control: image: "imlteam/manager-power-control:6.2.0-dev" hostname: "power-control" @@ -250,6 +258,8 @@ services: context: . dockerfile: ./power-control.dockerfile deploy: *default-deploy + environment: + - LOG_LEVEL=20 volumes: - "manager-config:/var/lib/chroma" syslog: @@ -259,6 +269,8 @@ services: context: . dockerfile: ./syslog.dockerfile deploy: *default-deploy + environment: + - LOG_LEVEL=20 volumes: - "manager-config:/var/lib/chroma" iml-warp-drive: @@ -314,6 +326,7 @@ services: - "manager-config:/var/lib/chroma" environment: - SERVER_HTTP_URL=gunicorn + - RUST_LOG=info iml-ostpool: image: "imlteam/iml-ostpool:6.2.0-dev" hostname: "iml-ostpool" @@ -332,6 +345,8 @@ services: context: ../ dockerfile: ./docker/iml-mailbox.dockerfile deploy: *default-deploy + environment: + - RUST_LOG=info volumes: - "mailbox:/var/spool/iml/mailbox" - "/sys/fs/cgroup:/sys/fs/cgroup:ro" @@ -360,6 +375,7 @@ services: - "manager-config:/var/lib/chroma" environment: - PROXY_HOST=iml-postoffice + - RUST_LOG=info iml-timer: image: "imlteam/iml-timer:6.2.0-dev" hostname: "iml-timer" @@ -367,6 +383,8 @@ services: context: ../ dockerfile: ./docker/iml-timer.dockerfile deploy: *default-deploy + environment: + - RUST_LOG=info volumes: - "/sys/fs/cgroup:/sys/fs/cgroup:ro" - "timer_run:/run" @@ -389,6 +407,7 @@ services: - USE_STRATAGEM - EXA_VERSION - INFLUXDB_IML_STATS_LONG_DURATION + - LOG_LEVEL=20 deploy: restart_policy: condition: on-failure diff --git a/iml-system-test-utils/src/lib.rs b/iml-system-test-utils/src/lib.rs index acd44144a9..e32dae9c17 100644 --- a/iml-system-test-utils/src/lib.rs +++ b/iml-system-test-utils/src/lib.rs @@ -273,14 +273,22 @@ impl Config { pub fn get_setup_config(&self) -> String { match &self.test_type { TestType::Rpm => format!( - r#"USE_STRATAGEM = {} -BRANDING = "{}""#, + r#"USE_STRATAGEM={} +BRANDING={} +LOG_LEVEL=10 +RUST_LOG=debug +NTP_SERVER_HOSTNAME=adm.local +"#, if self.use_stratagem { "True" } else { "False" }, self.branding.to_string() ), TestType::Docker => format!( r#"USE_STRATAGEM={} -BRANDING={}"#, +BRANDING={} +LOG_LEVEL=10 +RUST_LOG=debug +NTP_SERVER_HOSTNAME=10.73.10.1 +"#, self.use_stratagem, self.branding.to_string() ), @@ -447,7 +455,7 @@ pub async fn configure_iml(config: Config) -> Result { pub async fn deploy_servers(config: Config) -> Result { for (profile, hosts) in &config.profile_map { let host_ips = config.hosts_to_ips(&hosts); - for host in host_ips { + for host in &host_ips { tracing::debug!("pinging host to make sure it is up."); ssh::ssh_exec_cmd(host, "uname -r") .await? @@ -463,6 +471,7 @@ pub async fn deploy_servers(config: Config) -> Result { }; ssh::add_servers(&config.manager_ip, profile, hosts).await?; + ssh::enable_debug_on_hosts(&host_ips).await?; } vagrant::halt() @@ -616,12 +625,12 @@ pub async fn configure_rpm_setup(config: &Config) -> Result<(), CmdError> { let vagrant_path = canonicalize("../vagrant/").await?; let mut config_path = vagrant_path.clone(); - config_path.push("local_settings.py"); + config_path.push("overrides.conf"); let mut file = File::create(config_path).await?; file.write_all(config_content.as_bytes()).await?; - let mut vm_cmd: String = "sudo cp /vagrant/local_settings.py /usr/share/chroma-manager/".into(); + let mut vm_cmd: String = "mkdir -p /var/lib/chroma && sudo cp /vagrant/overrides.conf /var/lib/chroma".into(); if config.use_stratagem { let mut server_profile_path = vagrant_path.clone(); server_profile_path.push("stratagem-server.profile"); diff --git a/iml-system-test-utils/src/ssh.rs b/iml-system-test-utils/src/ssh.rs index 48dfdef95d..8271426a19 100644 --- a/iml-system-test-utils/src/ssh.rs +++ b/iml-system-test-utils/src/ssh.rs @@ -216,6 +216,12 @@ pub async fn wait_for_ntp_for_adm<'a, 'b>( ssh_script_parallel(hosts, "scripts/wait_for_ntp.sh", &["adm.local"]).await } +pub async fn enable_debug_on_hosts<'a, 'b>( + hosts: &'b [&'a str], +) -> Result, CmdError> { + ssh_script_parallel(hosts, "scripts/enable_debug.sh", &[]).await +} + pub async fn create_iml_diagnostics<'a, 'b>( hosts: Vec<&'a str>, prefix: &'a str, diff --git a/vagrant/Vagrantfile b/vagrant/Vagrantfile index e4c560f4a8..e6a8347562 100644 --- a/vagrant/Vagrantfile +++ b/vagrant/Vagrantfile @@ -467,6 +467,10 @@ Vagrant.configure('2') do |config| mkdir -p /mnt/mdt{0,1} SHELL + mds.vm.provision 'enable-debug', + type: 'shell', + run: 'never', + path: 'scripts/enable_debug.sh' end end @@ -793,6 +797,10 @@ Vagrant.configure('2') do |config| mkdir -p /mnt/ost{0..9} SHELL + oss.vm.provision 'enable-debug', + type: 'shell', + run: 'never', + path: 'scripts/enable_debug.sh' end end @@ -833,6 +841,10 @@ Vagrant.configure('2') do |config| type: 'shell', run: 'never', path: './scripts/configure_lustre_network.sh' + c.vm.provision 'enable-debug', + type: 'shell', + run: 'never', + path: 'scripts/enable_debug.sh' end end diff --git a/vagrant/scripts/enable_debug.sh b/vagrant/scripts/enable_debug.sh new file mode 100644 index 0000000000..e33bb541b9 --- /dev/null +++ b/vagrant/scripts/enable_debug.sh @@ -0,0 +1,2 @@ +#!/bin/bash +touch /tmp/chroma-agent-debug \ No newline at end of file diff --git a/vagrant/scripts/wait_for_ntp.sh b/vagrant/scripts/wait_for_ntp.sh index c231306174..ea1db38750 100755 --- a/vagrant/scripts/wait_for_ntp.sh +++ b/vagrant/scripts/wait_for_ntp.sh @@ -7,7 +7,7 @@ systemctl stop ntpd.service sntp -s "$NTP_SERVER" sntp -s "$NTP_SERVER" -systemctl restart ntpd.service +systemctl start ntpd.service until ntpstat; do