Create iml-report service

Fixes #2114

Signed-off-by: Nathaniel Clark <nclark@whamcloud.com>
Signed-off-by: Joe Grund <jgrund@whamcloud.io>

.copr/Makefile

@@ -38,11 +38,12 @@ srpm:
  mkdir -p ${TMPDIR}/_topdir/{SOURCES,SPECS}
  mkdir -p ${TMPDIR}/release/rust-iml
  PQ_LIB_DIR=/usr/pgsql-9.6/lib cargo build --release
- cp ${TARGET}/release/iml-{action-runner,agent,agent-comms,agent-daemon,api,device,journal,mailbox,ntp,ostpool,postoffice,sfa,stats,task-runner,warp-drive} \
+ cp ${TARGET}/release/iml-{action-runner,agent,agent-comms,agent-daemon,api,device,journal,mailbox,ntp,ostpool,postoffice,report,sfa,stats,task-runner,warp-drive} \
  iml-agent-comms.service \
  iml-mailbox.service \
  iml-postoffice.service \
  iml-ostpool.service \
+ iml-report.service \
  iml-api.service \
  iml-rust-stats.service \
  iml-device.service \
@@ -52,6 +53,7 @@ srpm:
  iml-action-runner.socket \
  iml-agent/systemd-units/* \
  iml-warp-drive/systemd-units/* \
+ iml-report.conf \
  iml-ntp.service \
  iml-sfa.service \
  ${TMPDIR}/release/rust-iml

Cargo.toml

@@ -16,6 +16,7 @@ members = [
  'iml-orm',
  'iml-postgres',
  'iml-rabbit',
+ 'iml-report',
  'iml-request-retry',
  'iml-services/iml-action-runner',
  'iml-services/iml-device',

chroma-manager.conf.template

@@ -228,7 +228,7 @@ server {
  if ($ssl_client_verify != SUCCESS) {
  return 401;
  }
- 
+
  client_max_body_size 0m;
 
  proxy_set_header X-SSL-Client-On $ssl_client_verify;
@@ -241,6 +241,38 @@ server {
  proxy_pass {{MAILBOX_PROXY_PASS}}/mailbox;
  }
 
+ location ~ /report/(.+)$ {
+ auth_request /auth;
+
+ types { } default_type application/octet-stream;
+ gzip on;
+ gzip_types application/octet-stream;
+
+ sendfile on;
+ tcp_nopush on;
+ tcp_nodelay on;
+ sendfile_max_chunk 1m;
+
+ alias {{REPORT_PATH}}/$1;
+ }
+
+ location /report {
+ if ($ssl_client_verify != SUCCESS) {
+ return 401;
+ }
+
+ client_max_body_size 0m;
+
+ proxy_set_header X-SSL-Client-On $ssl_client_verify;
+ proxy_set_header X-SSL-Client-Name $ssl_client_s_dn_cn;
+ proxy_set_header X-SSL-Client-Serial $ssl_client_serial;
+
+ proxy_set_header X-Forwarded-Host $host;
+ proxy_set_header X-Forwarded-Server $host;
+ proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+ proxy_pass {{REPORT_PROXY_PASS}}/report;
+ }
+
  location /agent/register {
  proxy_set_header X-Forwarded-Host $host;
  proxy_set_header X-Forwarded-Server $host;

chroma_core/management/commands/print-settings.py

@@ -28,6 +28,7 @@ def handle(self, *args, **kwargs):
  config = {
  "WARP_DRIVE_PORT": settings.WARP_DRIVE_PORT,
  "MAILBOX_PORT": settings.MAILBOX_PORT,
+ "REPORT_PORT": settings.REPORT_PORT,
  "DEVICE_AGGREGATOR_PORT": settings.DEVICE_AGGREGATOR_PORT,
  "HTTP_FRONTEND_PORT": settings.HTTP_FRONTEND_PORT,
  "HTTPS_FRONTEND_PORT": settings.HTTPS_FRONTEND_PORT,
@@ -39,6 +40,7 @@ def handle(self, *args, **kwargs):
  "IML_API_PROXY_PASS": settings.IML_API_PROXY_PASS,
  "WARP_DRIVE_PROXY_PASS": settings.WARP_DRIVE_PROXY_PASS,
  "MAILBOX_PROXY_PASS": settings.MAILBOX_PROXY_PASS,
+ "REPORT_PROXY_PASS": settings.REPORT_PROXY_PASS,
  "SSL_PATH": settings.SSL_PATH,
  "DEVICE_AGGREGATOR_PROXY_PASS": settings.DEVICE_AGGREGATOR_PROXY_PASS,
  "UPDATE_HANDLER_PROXY_PASS": settings.UPDATE_HANDLER_PROXY_PASS,
@@ -55,6 +57,7 @@ def handle(self, *args, **kwargs):
  "VERSION": settings.VERSION,
  "API_USER": API_USER,
  "API_KEY": API_KEY,
+ "REPORT_PATH": settings.REPORT_PATH,
  "PROXY_HOST": settings.PROXY_HOST,
  "INFLUXDB_IML_DB": settings.INFLUXDB_IML_DB,
  "INFLUXDB_STRATAGEM_SCAN_DB": settings.INFLUXDB_STRATAGEM_SCAN_DB,

chroma_core/services/job_scheduler/job_scheduler.py

@@ -1866,9 +1866,9 @@ def run_stratagem(self, mdts, fs_id, stratagem_data):
  "name": "{}-warn_fids-fids_expiring_soon".format(unique_id),
  "start": django.utils.timezone.now(),
  "state": "created",
- "single_runner": True,
+ "single_runner": False,
  "keep_failed": False,
- "args": {"report_file": "/tmp/expiring_fids-{}-{}.txt".format(filesystem.name, unique_id)},
+ "args": {"report_name": "expiring_fids-{}-{}.txt".format(filesystem.name, unique_id)},
  "actions": ["stratagem.warning"],
  }
  task = Task.objects.create(**task_data)

docker/docker-compose.yml

@@ -113,6 +113,7 @@ services:
  volumes:
  - "/etc/iml-docker/setup/branding:/var/lib/chroma/branding"
  - "manager-config:/var/lib/chroma"
+ - "report:/var/spool/iml/report"
  - "static-config1:/usr/lib/iml-manager"
  - "static-config2:/usr/lib/node_modules/@iml"
  ports:
@@ -223,6 +224,7 @@ services:
  deploy: *default-deploy
  volumes:
  - "manager-config:/var/lib/chroma"
+ - "report:/var/spool/iml/report"
  - "/root/.ssh:/root/.ssh"
  environment:
  - DEVICE_AGGREGATOR_URL=http://device:8008
@@ -362,13 +364,25 @@ services:
  context: ../
  dockerfile: ./docker/iml-mailbox.dockerfile
  deploy: *default-deploy
- environment:
- - RUST_LOG=info
  volumes:
  - "manager-config:/var/lib/chroma"
  environment:
  - PROXY_HOST=iml-mailbox
  - RUST_LOG=info
+ iml-report:
+ image: "imlteam/iml-report:6.2.0-dev"
+ hostname: "iml-report"
+ build:
+ context: ../
+ dockerfile: ./docker/iml-report.dockerfile
+ deploy: *default-deploy
+ environment:
+ - RUST_LOG=info
+ volumes:
+ - "report:/var/spool/iml/report"
+ - "/sys/fs/cgroup:/sys/fs/cgroup:ro"
+ - "report_run:/run"
+ - "manager-config:/var/lib/chroma"
  iml-stats:
  image: "imlteam/iml-stats:6.2.0-dev"
  hostname: "iml-stats"
@@ -441,6 +455,11 @@ volumes:
  driver_opts:
  type: tmpfs
  device: tmpfs
+ report:
+ report_run:
+ driver_opts:
+ type: tmpfs
+ device: tmpfs
  timer_run:
  driver_opts:
  type: tmpfs

docker/iml-report.dockerfile

@@ -0,0 +1,14 @@
+FROM rust-iml-base as builder
+
+FROM imlteam/systemd-base:6.2.0-dev
+COPY --from=builder /build/target/release/iml-report /bin/
+COPY docker/iml-report/iml-report.service /etc/systemd/system/
+COPY docker/iml-report/iml-report.conf /etc/systemd/system/
+COPY iml-report.conf /usr/lib/tmpfiles.d/
+COPY docker/wait-for-dependencies.sh /usr/local/bin/
+
+RUN systemctl enable iml-report
+
+ENTRYPOINT ["wait-for-dependencies.sh"]
+
+CMD ["/usr/sbin/init"]

docker/iml-report/iml-report.conf

@@ -0,0 +1 @@
+PROXY_HOST=iml-report

docker/iml-mailbox/iml-mailbox.service → docker/iml-report/iml-report.service

@@ -1,17 +1,17 @@
 [Unit]
-Description=IML Mailbox Service
+Description=IML Report Service
 After=syslog.target network.target
 
 
 [Service]
 Type=simple
 Environment=RUST_LOG=info
 EnvironmentFile=/var/lib/chroma/iml-settings.conf
-EnvironmentFile=/etc/systemd/system/iml-mailbox.conf
-ExecStart=/bin/iml-mailbox
+EnvironmentFile=/etc/systemd/system/iml-report.conf
+ExecStart=/bin/iml-report
 Restart=always
 StandardOutput=journal
 StandardError=journal
 
 [Install]
-WantedBy=multi-user.target
+WantedBy=multi-user.target

docker/setup-nginx

@@ -16,6 +16,8 @@ vars = {
  "GRAFANA_PROXY_PASS": "http://grafana:3000",
  "INFLUXDB_PROXY_PASS": "http://influxdb:8086",
  "IML_API_PROXY_PASS": "http://iml-api:8004",
+ "REPORT_PATH": "/var/spool/iml/report",
+ "REPORT_PROXY_PASS": "http://iml-report:8893",
  "INCLUDES": "include /etc/nginx/conf.d/iml-timer.extras;",
 }
 

iml-agent/src/action_plugins/stratagem/action_warning.rs

@@ -5,6 +5,7 @@
 use crate::{
  agent_error::{ImlAgentError, RequiredError},
  fidlist,
+ http_comms::streaming_client::send,
 };
 use futures::{
  channel::mpsc, future::join_all, sink::SinkExt, stream, StreamExt, TryFutureExt, TryStreamExt,
@@ -88,53 +89,50 @@ pub fn write_records(
 }
 
 pub async fn process_fids(
- (fsname_or_mntpath, task_args, fid_list): (String, HashMap<String, String>, Vec<FidItem>),
+ (fsname_or_mntpath, mut task_args, fid_list): (String, HashMap<String, String>, Vec<FidItem>),
 ) -> Result<Vec<FidError>, ImlAgentError> {
- let txt_path = task_args.get("report_file".into()).ok_or(RequiredError(
- "Task missing 'report_file' argument".to_string(),
+ let report_name = task_args.remove("report_name".into()).ok_or(RequiredError(
+ "Task missing 'report_name' argument".to_string(),
  ))?;
 
- let f = iml_fs::file_append_bytes(txt_path.into()).await?;
-
  let llapi = search_rootpath(fsname_or_mntpath).await?;
 
  let mntpt = llapi.mntpt();
 
  let (tx, rx) = mpsc::unbounded::<FidError>();
 
- tokio::spawn(
- stream::iter(fid_list)
- .chunks(1000)
- .map(|xs| Ok::<_, ImlAgentError>(xs.into_iter().collect()))
- .and_then(move |xs: Vec<_>| {
- let llapi = llapi.clone();
- let tx = tx.clone();
- async move {
- let xs = join_all(
- xs.into_iter()
- .map(move |x| item2path(llapi.clone(), x, tx.clone())),
- )
- .await;
-
- Ok(xs)
- }
- })
- .inspect(|_| debug!("Resolved 1000 Fids"))
- .map_ok(move |xs| {
- xs.into_iter()
- .filter_map(std::convert::identity)
- .map(|x| format!("{}/{}", mntpt, x))
- .collect()
- })
- .map_ok(|xs: Vec<String>| bytes::BytesMut::from(xs.join("\n").as_str()))
- .map_ok(|mut x: bytes::BytesMut| {
- if !x.is_empty() {
- x.extend_from_slice(b"\n");
- }
- x.freeze()
- })
- .forward(f.sink_err_into()),
- );
+ let s = stream::iter(fid_list)
+ .chunks(1000)
+ .map(|xs| Ok::<_, ImlAgentError>(xs.into_iter().collect()))
+ .and_then(move |xs: Vec<_>| {
+ let llapi = llapi.clone();
+ let tx = tx.clone();
+ async move {
+ let xs = join_all(
+ xs.into_iter()
+ .map(move |x| item2path(llapi.clone(), x, tx.clone())),
+ )
+ .await;
+
+ Ok(xs)
+ }
+ })
+ .inspect(|_| debug!("Resolved 1000 Fids"))
+ .map_ok(move |xs| {
+ xs.into_iter()
+ .filter_map(std::convert::identity)
+ .map(|x| format!("{}/{}", mntpt, x))
+ .collect()
+ })
+ .map_ok(|xs: Vec<String>| bytes::BytesMut::from(xs.join("\n").as_str()))
+ .map_ok(|mut x: bytes::BytesMut| {
+ if !x.is_empty() {
+ x.extend_from_slice(b"\n");
+ }
+ x.freeze()
+ });
+
+ tokio::spawn(send("report", report_name, s));
 
  Ok(rx.collect().await)
 }

iml-agent/src/action_plugins/stratagem/server.rs

@@ -2,7 +2,7 @@
 // Use of this source code is governed by a MIT-style
 // license that can be found in the LICENSE file.
 
-use crate::{agent_error::ImlAgentError, http_comms::mailbox_client};
+use crate::{agent_error::ImlAgentError, http_comms::streaming_client};
 use futures::{future, stream, StreamExt, TryStreamExt};
 use iml_cmd::{CheckedCommandExt, Command};
 use iml_fs::{read_file_to_end, stream_dir_lines, write_tempfile};
@@ -374,7 +374,7 @@ pub async fn stream_fidlists(mailbox_files: MailboxFiles) -> Result<(), ImlAgent
  Ok(format!("{{ \"fid\": \"{}\" }}\n", x).into())
  });
 
- mailbox_client::send(address.clone(), stream::iter(xs))
+ streaming_client::send("mailbox", address.clone(), stream::iter(xs))
  })
  });
 

iml-agent/src/daemon_plugins/postoffice.rs

@@ -6,7 +6,7 @@ use crate::{
  agent_error::ImlAgentError,
  daemon_plugins::{DaemonPlugin, Output},
  env,
- http_comms::mailbox_client::send,
+ http_comms::streaming_client::send,
 };
 use async_trait::async_trait;
 use futures::{
@@ -65,7 +65,7 @@ fn start_route(mailbox: String) -> Trigger {
  let stream = FramedRead::new(inbound, BytesCodec::new())
  .map_ok(bytes::BytesMut::freeze)
  .err_into();
- let transfer = send(mailbox.clone(), stream).map(|r| {
+ let transfer = send("mailbox", mailbox.clone(), stream).map(|r| {
  if let Err(e) = r {
  tracing::error!("Failed to transfer: {}", e);
  }

iml-agent/src/http_comms/mod.rs

@@ -4,5 +4,5 @@
 
 pub mod agent_client;
 pub mod crypto_client;
-pub mod mailbox_client;
 pub mod session;
+pub mod streaming_client;